superlocalmemory 2.8.5 → 3.0.0

package/src/graph/__init__.py

@@ -1,36 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""graph package - Knowledge Graph Clustering for SuperLocalMemory V2
-
-Re-exports all public classes, constants, and functions so that
-``from graph import GraphEngine`` (or any other symbol) works.
-"""
-from graph.constants import (
-    MAX_MEMORIES_FOR_GRAPH,
-    SKLEARN_AVAILABLE,
-    IGRAPH_AVAILABLE,
-    MEMORY_DIR,
-    DB_PATH,
-)
-from graph.entity_extractor import EntityExtractor, ClusterNamer
-from graph.edge_builder import EdgeBuilder
-from graph.cluster_builder import ClusterBuilder
-from graph.graph_core import GraphEngine
-from graph.cli import main
-
-__all__ = [
-    # Constants
-    "MAX_MEMORIES_FOR_GRAPH",
-    "SKLEARN_AVAILABLE",
-    "IGRAPH_AVAILABLE",
-    "MEMORY_DIR",
-    "DB_PATH",
-    # Classes
-    "EntityExtractor",
-    "ClusterNamer",
-    "EdgeBuilder",
-    "ClusterBuilder",
-    "GraphEngine",
-    # Functions
-    "main",
-]

package/src/graph/build_helpers.py

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Helper functions for the graph build process.
-
-Provides sampling and cleanup utilities used during full graph builds.
-"""
-from graph.constants import logger, MAX_MEMORIES_FOR_GRAPH
-
-
-def apply_sampling(cursor, memories, active_profile):
-    """Apply intelligent sampling if memory count exceeds cap.
-
-    Returns a (possibly truncated) list of memory tuples.
-    """
-    if len(memories) > MAX_MEMORIES_FOR_GRAPH:
-        logger.warning(
-            "Memory count (%d) exceeds graph cap (%d). Using intelligent sampling.",
-            len(memories), MAX_MEMORIES_FOR_GRAPH
-        )
-        recent_count = int(MAX_MEMORIES_FOR_GRAPH * 0.6)
-        important_count = int(MAX_MEMORIES_FOR_GRAPH * 0.4)
-
-        recent_memories = cursor.execute('''
-            SELECT id, content, summary FROM memories
-            WHERE profile = ? ORDER BY created_at DESC LIMIT ?
-        ''', (active_profile, recent_count)).fetchall()
-
-        important_memories = cursor.execute('''
-            SELECT id, content, summary FROM memories
-            WHERE profile = ? ORDER BY importance DESC, access_count DESC LIMIT ?
-        ''', (active_profile, important_count)).fetchall()
-
-        seen_ids = set()
-        sampled = []
-        for m in recent_memories + important_memories:
-            if m[0] not in seen_ids:
-                seen_ids.add(m[0])
-                sampled.append(m)
-        memories = sampled[:MAX_MEMORIES_FOR_GRAPH]
-        logger.info("Sampled %d memories for graph build", len(memories))
-
-    elif len(memories) > MAX_MEMORIES_FOR_GRAPH * 0.8:
-        logger.warning(
-            "Approaching graph cap: %d/%d memories (%.0f%%). "
-            "Consider running memory compression.",
-            len(memories), MAX_MEMORIES_FOR_GRAPH,
-            len(memories) / MAX_MEMORIES_FOR_GRAPH * 100
-        )
-    return memories
-
-
-def clear_profile_graph_data(cursor, conn, memories, active_profile):
-    """Clear existing graph data for a profile's memories."""
-    profile_memory_ids = [m[0] for m in memories]
-    if profile_memory_ids:
-        placeholders = ','.join('?' * len(profile_memory_ids))
-        cursor.execute(f'''
-            DELETE FROM graph_edges
-            WHERE source_memory_id IN ({placeholders})
-               OR target_memory_id IN ({placeholders})
-        ''', profile_memory_ids + profile_memory_ids)
-        cursor.execute(f'''
-            DELETE FROM graph_nodes WHERE memory_id IN ({placeholders})
-        ''', profile_memory_ids)
-    cursor.execute('''
-        DELETE FROM graph_clusters
-        WHERE id NOT IN (
-            SELECT DISTINCT cluster_id FROM memories WHERE cluster_id IS NOT NULL
-        )
-    ''')
-    cursor.execute('UPDATE memories SET cluster_id = NULL WHERE profile = ?',
-                  (active_profile,))
-    conn.commit()

package/src/graph/cli.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""CLI interface for manual graph operations.
-
-Provides a command-line interface for building graphs, viewing stats,
-finding related memories, inspecting clusters, and generating summaries.
-"""
-import json
-
-
-def main():
-    """CLI interface for manual graph operations."""
-    import argparse
-    from graph.graph_core import GraphEngine
-    from graph.cluster_builder import ClusterBuilder
-
-    parser = argparse.ArgumentParser(description='GraphEngine - Knowledge Graph Management')
-    parser.add_argument('command', choices=['build', 'stats', 'related', 'cluster', 'hierarchical', 'summaries'],
-                       help='Command to execute')
-    parser.add_argument('--memory-id', type=int, help='Memory ID for related/add commands')
-    parser.add_argument('--cluster-id', type=int, help='Cluster ID for cluster command')
-    parser.add_argument('--min-similarity', type=float, default=0.3,
-                       help='Minimum similarity for edges (default: 0.3)')
-    parser.add_argument('--hops', type=int, default=2, help='Max hops for related (default: 2)')
-
-    args = parser.parse_args()
-
-    engine = GraphEngine()
-
-    if args.command == 'build':
-        print("Building knowledge graph...")
-        stats = engine.build_graph(min_similarity=args.min_similarity)
-        print(json.dumps(stats, indent=2))
-
-    elif args.command == 'stats':
-        print("Graph Statistics:")
-        stats = engine.get_stats()
-        print(json.dumps(stats, indent=2))
-
-    elif args.command == 'related':
-        if not args.memory_id:
-            print("Error: --memory-id required for 'related' command")
-            return
-
-        print(f"Finding memories related to #{args.memory_id}...")
-        related = engine.get_related(args.memory_id, max_hops=args.hops)
-
-        if not related:
-            print("No related memories found")
-        else:
-            for idx, mem in enumerate(related, 1):
-                print(f"\n{idx}. Memory #{mem['id']} ({mem['hops']}-hop, weight={mem['weight']:.3f})")
-                print(f"   Relationship: {mem['relationship']}")
-                summary = mem['summary'] or '[No summary]'
-                print(f"   Summary: {summary[:100]}...")
-                if mem['shared_entities']:
-                    print(f"   Shared: {', '.join(mem['shared_entities'][:5])}")
-
-    elif args.command == 'cluster':
-        if not args.cluster_id:
-            print("Error: --cluster-id required for 'cluster' command")
-            return
-
-        print(f"Cluster #{args.cluster_id} members:")
-        members = engine.get_cluster_members(args.cluster_id)
-
-        for idx, mem in enumerate(members, 1):
-            print(f"\n{idx}. Memory #{mem['id']} (importance={mem['importance']})")
-            summary = mem['summary'] or '[No summary]'
-            print(f"   {summary[:100]}...")
-
-    elif args.command == 'hierarchical':
-        print("Running hierarchical sub-clustering...")
-        cluster_builder = ClusterBuilder(engine.db_path)
-        stats = cluster_builder.hierarchical_cluster()
-        print(json.dumps(stats, indent=2))
-
-    elif args.command == 'summaries':
-        print("Generating cluster summaries...")
-        cluster_builder = ClusterBuilder(engine.db_path)
-        count = cluster_builder.generate_cluster_summaries()
-        print(f"Generated summaries for {count} clusters")
-
-
-if __name__ == '__main__':
-    main()

package/src/graph/cluster_builder.py

@@ -1,188 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Community detection and cluster management for the graph engine.
-
-Implements Leiden algorithm based community detection. Hierarchical
-sub-clustering is delegated to the ``hierarchical`` module.
-"""
-import sqlite3
-import json
-from typing import List, Dict
-from collections import Counter
-
-from graph.constants import logger, IGRAPH_AVAILABLE, MEMORY_DIR
-from graph.cluster_summary import generate_cluster_summaries as _generate_summaries
-from graph.hierarchical import hierarchical_cluster as _hierarchical_cluster
-
-
-class ClusterBuilder:
-    """Detect memory communities using Leiden algorithm."""
-
-    def __init__(self, db_path):
-        """Initialize cluster builder."""
-        self.db_path = db_path
-
-    def _get_active_profile(self) -> str:
-        """Get the currently active profile name from config."""
-        config_file = MEMORY_DIR / "profiles.json"
-        if config_file.exists():
-            try:
-                with open(config_file, 'r') as f:
-                    config = json.load(f)
-                return config.get('active_profile', 'default')
-            except (json.JSONDecodeError, IOError):
-                pass
-        return 'default'
-
-    def detect_communities(self) -> int:
-        """
-        Run Leiden algorithm to find memory clusters (active profile only).
-
-        Returns:
-            Number of clusters created
-        """
-        if not IGRAPH_AVAILABLE:
-            logger.warning("igraph/leidenalg not installed. Graph clustering disabled. Install with: pip3 install python-igraph leidenalg")
-            return 0
-        import igraph as ig
-        import leidenalg
-
-        conn = sqlite3.connect(self.db_path)
-        cursor = conn.cursor()
-        active_profile = self._get_active_profile()
-
-        try:
-            # Load edges for active profile's memories only
-            edges = cursor.execute('''
-                SELECT ge.source_memory_id, ge.target_memory_id, ge.weight
-                FROM graph_edges ge
-                WHERE ge.source_memory_id IN (SELECT id FROM memories WHERE profile = ?)
-                  AND ge.target_memory_id IN (SELECT id FROM memories WHERE profile = ?)
-            ''', (active_profile, active_profile)).fetchall()
-
-            if not edges:
-                logger.warning("No edges found - cannot build clusters")
-                return 0
-
-            # Build memory ID mapping
-            memory_ids = set()
-            for source, target, _ in edges:
-                memory_ids.add(source)
-                memory_ids.add(target)
-
-            memory_ids = sorted(list(memory_ids))
-            memory_id_to_vertex = {mid: idx for idx, mid in enumerate(memory_ids)}
-            vertex_to_memory_id = {idx: mid for mid, idx in memory_id_to_vertex.items()}
-
-            # Create igraph graph
-            g = ig.Graph()
-            g.add_vertices(len(memory_ids))
-
-            edge_list = []
-            edge_weights = []
-            for source, target, weight in edges:
-                edge_list.append((memory_id_to_vertex[source], memory_id_to_vertex[target]))
-                edge_weights.append(weight)
-
-            g.add_edges(edge_list)
-
-            # Run Leiden algorithm
-            logger.info(f"Running Leiden on {len(memory_ids)} nodes, {len(edges)} edges")
-            partition = leidenalg.find_partition(
-                g, leidenalg.ModularityVertexPartition,
-                weights=edge_weights, n_iterations=100, seed=42
-            )
-
-            clusters_created = 0
-            for cluster_idx, community in enumerate(partition):
-                if len(community) < 2:
-                    continue
-
-                cluster_memory_ids = [vertex_to_memory_id[v] for v in community]
-                avg_importance = self._get_avg_importance(cursor, cluster_memory_ids)
-                cluster_name = self._generate_cluster_name(cursor, cluster_memory_ids)
-
-                result = cursor.execute('''
-                    INSERT INTO graph_clusters (name, member_count, avg_importance)
-                    VALUES (?, ?, ?)
-                ''', (cluster_name, len(cluster_memory_ids), avg_importance))
-
-                cluster_id = result.lastrowid
-                cursor.executemany('''
-                    UPDATE memories SET cluster_id = ? WHERE id = ?
-                ''', [(cluster_id, mid) for mid in cluster_memory_ids])
-
-                clusters_created += 1
-                logger.info(f"Cluster {cluster_id}: '{cluster_name}' ({len(cluster_memory_ids)} members)")
-
-            conn.commit()
-            logger.info(f"Created {clusters_created} clusters")
-            return clusters_created
-
-        except Exception as e:
-            logger.error(f"Community detection failed: {e}")
-            conn.rollback()
-            return 0
-        finally:
-            conn.close()
-
-    def _get_avg_importance(self, cursor, memory_ids: List[int]) -> float:
-        """Calculate average importance for cluster."""
-        placeholders = ','.join('?' * len(memory_ids))
-        result = cursor.execute(f'''
-            SELECT AVG(importance) FROM memories WHERE id IN ({placeholders})
-        ''', memory_ids).fetchone()
-        return result[0] if result and result[0] else 5.0
-
-    def _generate_cluster_name(self, cursor, memory_ids: List[int]) -> str:
-        """Generate cluster name from member entities (TF-IDF approach)."""
-        placeholders = ','.join('?' * len(memory_ids))
-        nodes = cursor.execute(f'''
-            SELECT entities FROM graph_nodes WHERE memory_id IN ({placeholders})
-        ''', memory_ids).fetchall()
-
-        all_entities = []
-        for node in nodes:
-            if node[0]:
-                all_entities.extend(json.loads(node[0]))
-
-        if not all_entities:
-            return f"Cluster (ID auto-assigned)"
-
-        entity_counts = Counter(all_entities)
-        top_entities = [e for e, _ in entity_counts.most_common(3)]
-
-        if len(top_entities) >= 2:
-            name = f"{top_entities[0].title()} & {top_entities[1].title()}"
-        elif len(top_entities) == 1:
-            name = f"{top_entities[0].title()} Contexts"
-        else:
-            name = "Mixed Contexts"
-
-        return name[:100]
-
-    def hierarchical_cluster(self, min_subcluster_size: int = 5, max_depth: int = 3) -> Dict[str, any]:
-        """
-        Run recursive Leiden clustering -- cluster the clusters.
-
-        Delegates to the hierarchical module.
-
-        Args:
-            min_subcluster_size: Minimum members to attempt sub-clustering (default 5)
-            max_depth: Maximum recursion depth (default 3)
-
-        Returns:
-            Dictionary with hierarchical clustering statistics
-        """
-        return _hierarchical_cluster(
-            self.db_path,
-            get_avg_importance_fn=self._get_avg_importance,
-            generate_cluster_name_fn=self._generate_cluster_name,
-            min_subcluster_size=min_subcluster_size,
-            max_depth=max_depth,
-        )
-
-    def generate_cluster_summaries(self) -> int:
-        """Generate TF-IDF structured summaries for all clusters."""
-        return _generate_summaries(self.db_path)

package/src/graph/cluster_summary.py

@@ -1,148 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Cluster summary generation for the graph engine.
-
-Generates TF-IDF structured summaries for graph clusters,
-analyzing member content to produce human-readable descriptions
-of each cluster's theme, key topics, and scope.
-"""
-import sqlite3
-import json
-from pathlib import Path
-from collections import Counter
-
-from graph.constants import logger, MEMORY_DIR
-
-
-def _get_active_profile() -> str:
-    """Get the currently active profile name from config."""
-    config_file = MEMORY_DIR / "profiles.json"
-    if config_file.exists():
-        try:
-            with open(config_file, 'r') as f:
-                config = json.load(f)
-            return config.get('active_profile', 'default')
-        except (json.JSONDecodeError, IOError):
-            pass
-    return 'default'
-
-
-def generate_cluster_summaries(db_path: Path) -> int:
-    """
-    Generate TF-IDF structured summaries for all clusters.
-
-    For each cluster, analyzes member content to produce a human-readable
-    summary describing the cluster's theme, key topics, and scope.
-
-    Returns:
-        Number of clusters with summaries generated
-    """
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    active_profile = _get_active_profile()
-
-    try:
-        # Get all clusters for this profile
-        cursor.execute('''
-            SELECT DISTINCT gc.id, gc.name, gc.member_count
-            FROM graph_clusters gc
-            JOIN memories m ON m.cluster_id = gc.id
-            WHERE m.profile = ?
-        ''', (active_profile,))
-        clusters = cursor.fetchall()
-
-        if not clusters:
-            return 0
-
-        summaries_generated = 0
-
-        for cluster_id, cluster_name, member_count in clusters:
-            summary = _build_cluster_summary(cursor, cluster_id, active_profile)
-            if summary:
-                cursor.execute('''
-                    UPDATE graph_clusters SET summary = ?, updated_at = CURRENT_TIMESTAMP
-                    WHERE id = ?
-                ''', (summary, cluster_id))
-                summaries_generated += 1
-                logger.info(f"Summary for cluster {cluster_id} ({cluster_name}): {summary[:80]}...")
-
-        conn.commit()
-        logger.info(f"Generated {summaries_generated} cluster summaries")
-        return summaries_generated
-
-    except Exception as e:
-        logger.error(f"Summary generation failed: {e}")
-        conn.rollback()
-        return 0
-    finally:
-        conn.close()
-
-
-def _build_cluster_summary(cursor, cluster_id: int, profile: str) -> str:
-    """Build a TF-IDF structured summary for a single cluster."""
-    # Get member content
-    cursor.execute('''
-        SELECT m.content, m.summary, m.tags, m.category, m.project_name
-        FROM memories m
-        WHERE m.cluster_id = ? AND m.profile = ?
-    ''', (cluster_id, profile))
-    members = cursor.fetchall()
-
-    if not members:
-        return ""
-
-    # Collect entities from graph nodes
-    cursor.execute('''
-        SELECT gn.entities
-        FROM graph_nodes gn
-        JOIN memories m ON gn.memory_id = m.id
-        WHERE m.cluster_id = ? AND m.profile = ?
-    ''', (cluster_id, profile))
-    all_entities = []
-    for row in cursor.fetchall():
-        if row[0]:
-            try:
-                all_entities.extend(json.loads(row[0]))
-            except (json.JSONDecodeError, TypeError):
-                pass
-
-    # Top entities by frequency (TF-IDF already extracted these)
-    entity_counts = Counter(all_entities)
-    top_entities = [e for e, _ in entity_counts.most_common(5)]
-
-    # Collect unique projects and categories
-    projects = set()
-    categories = set()
-    for m in members:
-        if m[3]:  # category
-            categories.add(m[3])
-        if m[4]:  # project_name
-            projects.add(m[4])
-
-    # Build structured summary
-    parts = []
-
-    # Theme from top entities
-    if top_entities:
-        parts.append(f"Key topics: {', '.join(top_entities[:5])}")
-
-    # Scope
-    if projects:
-        parts.append(f"Projects: {', '.join(sorted(projects)[:3])}")
-    if categories:
-        parts.append(f"Categories: {', '.join(sorted(categories)[:3])}")
-
-    # Size context
-    parts.append(f"{len(members)} memories")
-
-    # Check for hierarchical context
-    cursor.execute('SELECT parent_cluster_id FROM graph_clusters WHERE id = ?', (cluster_id,))
-    parent_row = cursor.fetchone()
-    if parent_row and parent_row[0]:
-        cursor.execute('SELECT name FROM graph_clusters WHERE id = ?', (parent_row[0],))
-        parent_name_row = cursor.fetchone()
-        if parent_name_row:
-            parts.append(f"Sub-cluster of: {parent_name_row[0]}")
-
-    return " | ".join(parts)

package/src/graph/constants.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Shared constants, imports, and configuration for the graph engine modules.
-"""
-# SECURITY: Graph build limits to prevent resource exhaustion
-MAX_MEMORIES_FOR_GRAPH = 10000
-
-import sqlite3
-import json
-import time
-import logging
-from datetime import datetime
-from pathlib import Path
-from typing import List, Dict, Optional, Tuple, Set
-from collections import Counter
-
-# Core dependencies
-try:
-    from sklearn.feature_extraction.text import TfidfVectorizer
-    from sklearn.metrics.pairwise import cosine_similarity
-    import numpy as np
-    SKLEARN_AVAILABLE = True
-except ImportError:
-    SKLEARN_AVAILABLE = False
-    raise ImportError("scikit-learn is required. Install: pip install scikit-learn")
-
-# Graph dependencies - lazy import to avoid conflicts with compression module
-IGRAPH_AVAILABLE = False
-try:
-    # Import only when needed to avoid module conflicts
-    import importlib
-    ig_module = importlib.import_module('igraph')
-    leiden_module = importlib.import_module('leidenalg')
-    IGRAPH_AVAILABLE = True
-except ImportError:
-    pass  # Will raise error when building clusters if not available
-
-# Setup logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger('graph_engine')
-
-MEMORY_DIR = Path.home() / ".claude-memory"
-DB_PATH = MEMORY_DIR / "memory.db"