superlocalmemory 2.7.6 → 2.8.0

package/src/embeddings/model_loader.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Model loading and backend encoder methods for EmbeddingEngine.
+"""
+import time
+import logging
+from typing import List
+
+import numpy as np
+
+from embeddings.constants import (
+    SENTENCE_TRANSFORMERS_AVAILABLE,
+    SKLEARN_AVAILABLE,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ModelLoaderMixin:
+    """
+    Mixin that handles model initialization and raw encoding backends.
+
+    Expects the host class to have:
+        - self.use_transformers: bool
+        - self.model_cache_path: Path
+        - self.model_name: str
+        - self.device: str
+        - self.model: Optional[SentenceTransformer]
+        - self.dimension: int
+        - self.tfidf_vectorizer
+        - self.tfidf_fitted: bool
+    """
+
+    def _load_model(self):
+        """Load sentence transformer model or fallback to TF-IDF."""
+        if not self.use_transformers:
+            logger.warning(
+                "sentence-transformers unavailable. Install with: "
+                "pip install sentence-transformers"
+            )
+            self._init_fallback()
+            return
+
+        try:
+            from sentence_transformers import SentenceTransformer
+
+            # Create model cache directory
+            self.model_cache_path.mkdir(parents=True, exist_ok=True)
+
+            logger.info(f"Loading model: {self.model_name}")
+            start_time = time.time()
+
+            # Load model with local cache
+            self.model = SentenceTransformer(
+                self.model_name,
+                device=self.device,
+                cache_folder=str(self.model_cache_path)
+            )
+
+            # Get actual dimension
+            self.dimension = self.model.get_sentence_embedding_dimension()
+
+            elapsed = time.time() - start_time
+            logger.info(
+                f"Loaded {self.model_name} ({self.dimension}D) in {elapsed:.2f}s"
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to load sentence transformer: {e}")
+            logger.info("Falling back to TF-IDF")
+            self.use_transformers = False
+            self._init_fallback()
+
+    def _init_fallback(self):
+        """Initialize TF-IDF fallback."""
+        if not SKLEARN_AVAILABLE:
+            logger.error(
+                "sklearn unavailable - no fallback available. "
+                "Install: pip install scikit-learn"
+            )
+            return
+
+        from sklearn.feature_extraction.text import TfidfVectorizer
+
+        logger.info("Using TF-IDF fallback (dimension will be dynamic)")
+        self.tfidf_vectorizer = TfidfVectorizer(
+            max_features=384,  # Match sentence transformer dimension
+            stop_words='english',
+            ngram_range=(1, 2),
+            min_df=1
+        )
+        self.dimension = 384
+
+    def _encode_transformer(
+        self,
+        texts: List[str],
+        batch_size: int,
+        show_progress: bool
+    ) -> np.ndarray:
+        """Generate embeddings using sentence transformer."""
+        try:
+            start_time = time.time()
+
+            embeddings = self.model.encode(
+                texts,
+                batch_size=batch_size,
+                show_progress_bar=show_progress,
+                convert_to_numpy=True,
+                normalize_embeddings=False  # We'll normalize separately
+            )
+
+            elapsed = time.time() - start_time
+            rate = len(texts) / elapsed if elapsed > 0 else 0
+            logger.debug(f"Encoded {len(texts)} texts in {elapsed:.2f}s ({rate:.0f} texts/sec)")
+
+            return embeddings
+
+        except Exception as e:
+            logger.error(f"Transformer encoding failed: {e}")
+            raise
+
+    def _encode_tfidf(self, texts: List[str]) -> np.ndarray:
+        """Generate embeddings using TF-IDF fallback."""
+        try:
+            if not self.tfidf_fitted:
+                # Fit on first use
+                logger.info("Fitting TF-IDF vectorizer...")
+                self.tfidf_vectorizer.fit(texts)
+                self.tfidf_fitted = True
+
+            embeddings = self.tfidf_vectorizer.transform(texts).toarray()
+
+            # Pad or truncate to target dimension
+            if embeddings.shape[1] < self.dimension:
+                padding = np.zeros((embeddings.shape[0], self.dimension - embeddings.shape[1]))
+                embeddings = np.hstack([embeddings, padding])
+            elif embeddings.shape[1] > self.dimension:
+                embeddings = embeddings[:, :self.dimension]
+
+            return embeddings
+
+        except Exception as e:
+            logger.error(f"TF-IDF encoding failed: {e}")
+            raise

package/src/event_bus.py

@@ -1,16 +1,6 @@
 #!/usr/bin/env python3
-"""
-SuperLocalMemory V2 - Event Bus
-Copyright (c) 2026 Varun Pratap Bhardwaj
-Licensed under MIT License
-
-Repository: https://github.com/varun369/SuperLocalMemoryV2
-Author: Varun Pratap Bhardwaj (Solution Architect)
-
-NOTICE: This software is protected by MIT License.
-Attribution must be preserved in all copies or derivatives.
-"""
-
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
 """
 EventBus — Real-time event broadcasting for memory operations.
 
@@ -222,7 +212,7 @@ class EventBus:
             payload: Event-specific data (dict, serialized to JSON)
             memory_id: Associated memory ID (if applicable)
             source_agent: Agent that triggered the event
-            source_protocol: Protocol used (mcp, cli, rest, python, a2a)
+            source_protocol: Protocol used (mcp, cli, rest, python)
             importance: Event importance 1-10 (affects retention)
 
         Returns:

package/src/graph/__init__.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""graph package - Knowledge Graph Clustering for SuperLocalMemory V2
+
+Re-exports all public classes, constants, and functions so that
+``from graph import GraphEngine`` (or any other symbol) works.
+"""
+from graph.constants import (
+    MAX_MEMORIES_FOR_GRAPH,
+    SKLEARN_AVAILABLE,
+    IGRAPH_AVAILABLE,
+    MEMORY_DIR,
+    DB_PATH,
+)
+from graph.entity_extractor import EntityExtractor, ClusterNamer
+from graph.edge_builder import EdgeBuilder
+from graph.cluster_builder import ClusterBuilder
+from graph.graph_core import GraphEngine
+from graph.cli import main
+
+__all__ = [
+    # Constants
+    "MAX_MEMORIES_FOR_GRAPH",
+    "SKLEARN_AVAILABLE",
+    "IGRAPH_AVAILABLE",
+    "MEMORY_DIR",
+    "DB_PATH",
+    # Classes
+    "EntityExtractor",
+    "ClusterNamer",
+    "EdgeBuilder",
+    "ClusterBuilder",
+    "GraphEngine",
+    # Functions
+    "main",
+]

package/src/graph/build_helpers.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Helper functions for the graph build process.
+
+Provides sampling and cleanup utilities used during full graph builds.
+"""
+from graph.constants import logger, MAX_MEMORIES_FOR_GRAPH
+
+
+def apply_sampling(cursor, memories, active_profile):
+    """Apply intelligent sampling if memory count exceeds cap.
+
+    Returns a (possibly truncated) list of memory tuples.
+    """
+    if len(memories) > MAX_MEMORIES_FOR_GRAPH:
+        logger.warning(
+            "Memory count (%d) exceeds graph cap (%d). Using intelligent sampling.",
+            len(memories), MAX_MEMORIES_FOR_GRAPH
+        )
+        recent_count = int(MAX_MEMORIES_FOR_GRAPH * 0.6)
+        important_count = int(MAX_MEMORIES_FOR_GRAPH * 0.4)
+
+        recent_memories = cursor.execute('''
+            SELECT id, content, summary FROM memories
+            WHERE profile = ? ORDER BY created_at DESC LIMIT ?
+        ''', (active_profile, recent_count)).fetchall()
+
+        important_memories = cursor.execute('''
+            SELECT id, content, summary FROM memories
+            WHERE profile = ? ORDER BY importance DESC, access_count DESC LIMIT ?
+        ''', (active_profile, important_count)).fetchall()
+
+        seen_ids = set()
+        sampled = []
+        for m in recent_memories + important_memories:
+            if m[0] not in seen_ids:
+                seen_ids.add(m[0])
+                sampled.append(m)
+        memories = sampled[:MAX_MEMORIES_FOR_GRAPH]
+        logger.info("Sampled %d memories for graph build", len(memories))
+
+    elif len(memories) > MAX_MEMORIES_FOR_GRAPH * 0.8:
+        logger.warning(
+            "Approaching graph cap: %d/%d memories (%.0f%%). "
+            "Consider running memory compression.",
+            len(memories), MAX_MEMORIES_FOR_GRAPH,
+            len(memories) / MAX_MEMORIES_FOR_GRAPH * 100
+        )
+    return memories
+
+
+def clear_profile_graph_data(cursor, conn, memories, active_profile):
+    """Clear existing graph data for a profile's memories."""
+    profile_memory_ids = [m[0] for m in memories]
+    if profile_memory_ids:
+        placeholders = ','.join('?' * len(profile_memory_ids))
+        cursor.execute(f'''
+            DELETE FROM graph_edges
+            WHERE source_memory_id IN ({placeholders})
+               OR target_memory_id IN ({placeholders})
+        ''', profile_memory_ids + profile_memory_ids)
+        cursor.execute(f'''
+            DELETE FROM graph_nodes WHERE memory_id IN ({placeholders})
+        ''', profile_memory_ids)
+    cursor.execute('''
+        DELETE FROM graph_clusters
+        WHERE id NOT IN (
+            SELECT DISTINCT cluster_id FROM memories WHERE cluster_id IS NOT NULL
+        )
+    ''')
+    cursor.execute('UPDATE memories SET cluster_id = NULL WHERE profile = ?',
+                  (active_profile,))
+    conn.commit()

package/src/graph/cli.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""CLI interface for manual graph operations.
+
+Provides a command-line interface for building graphs, viewing stats,
+finding related memories, inspecting clusters, and generating summaries.
+"""
+import json
+
+
+def main():
+    """CLI interface for manual graph operations."""
+    import argparse
+    from graph.graph_core import GraphEngine
+    from graph.cluster_builder import ClusterBuilder
+
+    parser = argparse.ArgumentParser(description='GraphEngine - Knowledge Graph Management')
+    parser.add_argument('command', choices=['build', 'stats', 'related', 'cluster', 'hierarchical', 'summaries'],
+                       help='Command to execute')
+    parser.add_argument('--memory-id', type=int, help='Memory ID for related/add commands')
+    parser.add_argument('--cluster-id', type=int, help='Cluster ID for cluster command')
+    parser.add_argument('--min-similarity', type=float, default=0.3,
+                       help='Minimum similarity for edges (default: 0.3)')
+    parser.add_argument('--hops', type=int, default=2, help='Max hops for related (default: 2)')
+
+    args = parser.parse_args()
+
+    engine = GraphEngine()
+
+    if args.command == 'build':
+        print("Building knowledge graph...")
+        stats = engine.build_graph(min_similarity=args.min_similarity)
+        print(json.dumps(stats, indent=2))
+
+    elif args.command == 'stats':
+        print("Graph Statistics:")
+        stats = engine.get_stats()
+        print(json.dumps(stats, indent=2))
+
+    elif args.command == 'related':
+        if not args.memory_id:
+            print("Error: --memory-id required for 'related' command")
+            return
+
+        print(f"Finding memories related to #{args.memory_id}...")
+        related = engine.get_related(args.memory_id, max_hops=args.hops)
+
+        if not related:
+            print("No related memories found")
+        else:
+            for idx, mem in enumerate(related, 1):
+                print(f"\n{idx}. Memory #{mem['id']} ({mem['hops']}-hop, weight={mem['weight']:.3f})")
+                print(f"   Relationship: {mem['relationship']}")
+                summary = mem['summary'] or '[No summary]'
+                print(f"   Summary: {summary[:100]}...")
+                if mem['shared_entities']:
+                    print(f"   Shared: {', '.join(mem['shared_entities'][:5])}")
+
+    elif args.command == 'cluster':
+        if not args.cluster_id:
+            print("Error: --cluster-id required for 'cluster' command")
+            return
+
+        print(f"Cluster #{args.cluster_id} members:")
+        members = engine.get_cluster_members(args.cluster_id)
+
+        for idx, mem in enumerate(members, 1):
+            print(f"\n{idx}. Memory #{mem['id']} (importance={mem['importance']})")
+            summary = mem['summary'] or '[No summary]'
+            print(f"   {summary[:100]}...")
+
+    elif args.command == 'hierarchical':
+        print("Running hierarchical sub-clustering...")
+        cluster_builder = ClusterBuilder(engine.db_path)
+        stats = cluster_builder.hierarchical_cluster()
+        print(json.dumps(stats, indent=2))
+
+    elif args.command == 'summaries':
+        print("Generating cluster summaries...")
+        cluster_builder = ClusterBuilder(engine.db_path)
+        count = cluster_builder.generate_cluster_summaries()
+        print(f"Generated summaries for {count} clusters")
+
+
+if __name__ == '__main__':
+    main()

package/src/graph/cluster_builder.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Community detection and cluster management for the graph engine.
+
+Implements Leiden algorithm based community detection. Hierarchical
+sub-clustering is delegated to the ``hierarchical`` module.
+"""
+import sqlite3
+import json
+from typing import List, Dict
+from collections import Counter
+
+from graph.constants import logger, IGRAPH_AVAILABLE, MEMORY_DIR
+from graph.cluster_summary import generate_cluster_summaries as _generate_summaries
+from graph.hierarchical import hierarchical_cluster as _hierarchical_cluster
+
+
+class ClusterBuilder:
+    """Detect memory communities using Leiden algorithm."""
+
+    def __init__(self, db_path):
+        """Initialize cluster builder."""
+        self.db_path = db_path
+
+    def _get_active_profile(self) -> str:
+        """Get the currently active profile name from config."""
+        config_file = MEMORY_DIR / "profiles.json"
+        if config_file.exists():
+            try:
+                with open(config_file, 'r') as f:
+                    config = json.load(f)
+                return config.get('active_profile', 'default')
+            except (json.JSONDecodeError, IOError):
+                pass
+        return 'default'
+
+    def detect_communities(self) -> int:
+        """
+        Run Leiden algorithm to find memory clusters (active profile only).
+
+        Returns:
+            Number of clusters created
+        """
+        if not IGRAPH_AVAILABLE:
+            logger.warning("igraph/leidenalg not installed. Graph clustering disabled. Install with: pip3 install python-igraph leidenalg")
+            return 0
+        import igraph as ig
+        import leidenalg
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        active_profile = self._get_active_profile()
+
+        try:
+            # Load edges for active profile's memories only
+            edges = cursor.execute('''
+                SELECT ge.source_memory_id, ge.target_memory_id, ge.weight
+                FROM graph_edges ge
+                WHERE ge.source_memory_id IN (SELECT id FROM memories WHERE profile = ?)
+                  AND ge.target_memory_id IN (SELECT id FROM memories WHERE profile = ?)
+            ''', (active_profile, active_profile)).fetchall()
+
+            if not edges:
+                logger.warning("No edges found - cannot build clusters")
+                return 0
+
+            # Build memory ID mapping
+            memory_ids = set()
+            for source, target, _ in edges:
+                memory_ids.add(source)
+                memory_ids.add(target)
+
+            memory_ids = sorted(list(memory_ids))
+            memory_id_to_vertex = {mid: idx for idx, mid in enumerate(memory_ids)}
+            vertex_to_memory_id = {idx: mid for mid, idx in memory_id_to_vertex.items()}
+
+            # Create igraph graph
+            g = ig.Graph()
+            g.add_vertices(len(memory_ids))
+
+            edge_list = []
+            edge_weights = []
+            for source, target, weight in edges:
+                edge_list.append((memory_id_to_vertex[source], memory_id_to_vertex[target]))
+                edge_weights.append(weight)
+
+            g.add_edges(edge_list)
+
+            # Run Leiden algorithm
+            logger.info(f"Running Leiden on {len(memory_ids)} nodes, {len(edges)} edges")
+            partition = leidenalg.find_partition(
+                g, leidenalg.ModularityVertexPartition,
+                weights=edge_weights, n_iterations=100, seed=42
+            )
+
+            clusters_created = 0
+            for cluster_idx, community in enumerate(partition):
+                if len(community) < 2:
+                    continue
+
+                cluster_memory_ids = [vertex_to_memory_id[v] for v in community]
+                avg_importance = self._get_avg_importance(cursor, cluster_memory_ids)
+                cluster_name = self._generate_cluster_name(cursor, cluster_memory_ids)
+
+                result = cursor.execute('''
+                    INSERT INTO graph_clusters (name, member_count, avg_importance)
+                    VALUES (?, ?, ?)
+                ''', (cluster_name, len(cluster_memory_ids), avg_importance))
+
+                cluster_id = result.lastrowid
+                cursor.executemany('''
+                    UPDATE memories SET cluster_id = ? WHERE id = ?
+                ''', [(cluster_id, mid) for mid in cluster_memory_ids])
+
+                clusters_created += 1
+                logger.info(f"Cluster {cluster_id}: '{cluster_name}' ({len(cluster_memory_ids)} members)")
+
+            conn.commit()
+            logger.info(f"Created {clusters_created} clusters")
+            return clusters_created
+
+        except Exception as e:
+            logger.error(f"Community detection failed: {e}")
+            conn.rollback()
+            return 0
+        finally:
+            conn.close()
+
+    def _get_avg_importance(self, cursor, memory_ids: List[int]) -> float:
+        """Calculate average importance for cluster."""
+        placeholders = ','.join('?' * len(memory_ids))
+        result = cursor.execute(f'''
+            SELECT AVG(importance) FROM memories WHERE id IN ({placeholders})
+        ''', memory_ids).fetchone()
+        return result[0] if result and result[0] else 5.0
+
+    def _generate_cluster_name(self, cursor, memory_ids: List[int]) -> str:
+        """Generate cluster name from member entities (TF-IDF approach)."""
+        placeholders = ','.join('?' * len(memory_ids))
+        nodes = cursor.execute(f'''
+            SELECT entities FROM graph_nodes WHERE memory_id IN ({placeholders})
+        ''', memory_ids).fetchall()
+
+        all_entities = []
+        for node in nodes:
+            if node[0]:
+                all_entities.extend(json.loads(node[0]))
+
+        if not all_entities:
+            return f"Cluster (ID auto-assigned)"
+
+        entity_counts = Counter(all_entities)
+        top_entities = [e for e, _ in entity_counts.most_common(3)]
+
+        if len(top_entities) >= 2:
+            name = f"{top_entities[0].title()} & {top_entities[1].title()}"
+        elif len(top_entities) == 1:
+            name = f"{top_entities[0].title()} Contexts"
+        else:
+            name = "Mixed Contexts"
+
+        return name[:100]
+
+    def hierarchical_cluster(self, min_subcluster_size: int = 5, max_depth: int = 3) -> Dict[str, any]:
+        """
+        Run recursive Leiden clustering -- cluster the clusters.
+
+        Delegates to the hierarchical module.
+
+        Args:
+            min_subcluster_size: Minimum members to attempt sub-clustering (default 5)
+            max_depth: Maximum recursion depth (default 3)
+
+        Returns:
+            Dictionary with hierarchical clustering statistics
+        """
+        return _hierarchical_cluster(
+            self.db_path,
+            get_avg_importance_fn=self._get_avg_importance,
+            generate_cluster_name_fn=self._generate_cluster_name,
+            min_subcluster_size=min_subcluster_size,
+            max_depth=max_depth,
+        )
+
+    def generate_cluster_summaries(self) -> int:
+        """Generate TF-IDF structured summaries for all clusters."""
+        return _generate_summaries(self.db_path)