superlocalmemory 2.7.5 → 2.8.0

package/src/graph/graph_search.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Graph traversal and query operations.
+
+Provides graph traversal (get_related), cluster membership queries,
+and graph statistics collection for the active profile.
+"""
+import sqlite3
+import json
+from pathlib import Path
+from typing import List, Dict
+
+from graph.constants import logger, MEMORY_DIR
+
+
+def _get_active_profile() -> str:
+    """Get the currently active profile name from config."""
+    config_file = MEMORY_DIR / "profiles.json"
+    if config_file.exists():
+        try:
+            with open(config_file, 'r') as f:
+                config = json.load(f)
+            return config.get('active_profile', 'default')
+        except (json.JSONDecodeError, IOError):
+            pass
+    return 'default'
+
+
+def get_related(db_path: Path, memory_id: int, max_hops: int = 2) -> List[Dict]:
+    """
+    Get memories connected to this memory via graph edges (active profile only).
+
+    Args:
+        db_path: Path to SQLite database
+        memory_id: Source memory ID
+        max_hops: Maximum traversal depth (1 or 2)
+
+    Returns:
+        List of related memory dictionaries
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    active_profile = _get_active_profile()
+
+    try:
+        # Get 1-hop neighbors (filtered to active profile)
+        edges = cursor.execute('''
+            SELECT ge.target_memory_id, ge.relationship_type, ge.weight, ge.shared_entities
+            FROM graph_edges ge
+            JOIN memories m ON ge.target_memory_id = m.id
+            WHERE ge.source_memory_id = ? AND m.profile = ?
+            UNION
+            SELECT ge.source_memory_id, ge.relationship_type, ge.weight, ge.shared_entities
+            FROM graph_edges ge
+            JOIN memories m ON ge.source_memory_id = m.id
+            WHERE ge.target_memory_id = ? AND m.profile = ?
+        ''', (memory_id, active_profile, memory_id, active_profile)).fetchall()
+
+        results = []
+        seen_ids = {memory_id}
+
+        for target_id, rel_type, weight, shared_entities in edges:
+            if target_id in seen_ids:
+                continue
+
+            seen_ids.add(target_id)
+
+            # Get memory details
+            memory = cursor.execute('''
+                SELECT id, summary, importance, tags
+                FROM memories WHERE id = ?
+            ''', (target_id,)).fetchone()
+
+            if memory:
+                results.append({
+                    'id': memory[0],
+                    'summary': memory[1],
+                    'importance': memory[2],
+                    'tags': json.loads(memory[3]) if memory[3] else [],
+                    'relationship': rel_type,
+                    'weight': weight,
+                    'shared_entities': json.loads(shared_entities) if shared_entities else [],
+                    'hops': 1
+                })
+
+        # If max_hops == 2, get 2-hop neighbors
+        if max_hops >= 2:
+            for result in results[:]:  # Copy to avoid modification during iteration
+                second_hop = cursor.execute('''
+                    SELECT target_memory_id, relationship_type, weight
+                    FROM graph_edges
+                    WHERE source_memory_id = ?
+                    UNION
+                    SELECT source_memory_id, relationship_type, weight
+                    FROM graph_edges
+                    WHERE target_memory_id = ?
+                ''', (result['id'], result['id'])).fetchall()
+
+                for target_id, rel_type, weight in second_hop:
+                    if target_id in seen_ids:
+                        continue
+
+                    seen_ids.add(target_id)
+
+                    memory = cursor.execute('''
+                        SELECT id, summary, importance, tags
+                        FROM memories WHERE id = ?
+                    ''', (target_id,)).fetchone()
+
+                    if memory:
+                        results.append({
+                            'id': memory[0],
+                            'summary': memory[1],
+                            'importance': memory[2],
+                            'tags': json.loads(memory[3]) if memory[3] else [],
+                            'relationship': rel_type,
+                            'weight': weight,
+                            'shared_entities': [],
+                            'hops': 2
+                        })
+
+        # Sort by weight (strongest connections first)
+        results.sort(key=lambda x: (-x['hops'], -x['weight']))
+
+        return results
+
+    finally:
+        conn.close()
+
+
+def get_cluster_members(db_path: Path, cluster_id: int) -> List[Dict]:
+    """
+    Get all memories in a cluster (filtered by active profile).
+
+    Args:
+        db_path: Path to SQLite database
+        cluster_id: Cluster ID
+
+    Returns:
+        List of memory dictionaries
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    active_profile = _get_active_profile()
+
+    try:
+        memories = cursor.execute('''
+            SELECT id, summary, importance, tags, created_at
+            FROM memories
+            WHERE cluster_id = ? AND profile = ?
+            ORDER BY importance DESC
+        ''', (cluster_id, active_profile)).fetchall()
+
+        return [
+            {
+                'id': m[0],
+                'summary': m[1],
+                'importance': m[2],
+                'tags': json.loads(m[3]) if m[3] else [],
+                'created_at': m[4]
+            }
+            for m in memories
+        ]
+
+    finally:
+        conn.close()
+
+
+def get_stats(db_path: Path) -> Dict[str, any]:
+    """Get graph statistics for the active profile."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    active_profile = _get_active_profile()
+
+    try:
+        # Count nodes for active profile's memories
+        nodes = cursor.execute('''
+            SELECT COUNT(*) FROM graph_nodes
+            WHERE memory_id IN (SELECT id FROM memories WHERE profile = ?)
+        ''', (active_profile,)).fetchone()[0]
+
+        # Count edges where at least one end is in active profile
+        edges = cursor.execute('''
+            SELECT COUNT(*) FROM graph_edges
+            WHERE source_memory_id IN (SELECT id FROM memories WHERE profile = ?)
+        ''', (active_profile,)).fetchone()[0]
+
+        # Clusters that have members in active profile
+        clusters = cursor.execute('''
+            SELECT COUNT(DISTINCT cluster_id) FROM memories
+            WHERE cluster_id IS NOT NULL AND profile = ?
+        ''', (active_profile,)).fetchone()[0]
+
+        # Cluster breakdown for active profile (including hierarchy)
+        cluster_info = cursor.execute('''
+            SELECT gc.name, gc.member_count, gc.avg_importance,
+                   gc.summary, gc.parent_cluster_id, gc.depth
+            FROM graph_clusters gc
+            WHERE gc.id IN (
+                SELECT DISTINCT cluster_id FROM memories
+                WHERE cluster_id IS NOT NULL AND profile = ?
+            )
+            ORDER BY gc.depth ASC, gc.member_count DESC
+            LIMIT 20
+        ''', (active_profile,)).fetchall()
+
+        # Count hierarchical depth
+        max_depth = max((c[5] or 0 for c in cluster_info), default=0) if cluster_info else 0
+
+        return {
+            'profile': active_profile,
+            'nodes': nodes,
+            'edges': edges,
+            'clusters': clusters,
+            'max_depth': max_depth,
+            'top_clusters': [
+                {
+                    'name': c[0],
+                    'members': c[1],
+                    'avg_importance': round(c[2], 1) if c[2] else 5.0,
+                    'summary': c[3],
+                    'parent_cluster_id': c[4],
+                    'depth': c[5] or 0
+                }
+                for c in cluster_info
+            ]
+        }
+
+    finally:
+        conn.close()

package/src/graph/hierarchical.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Hierarchical sub-clustering for the graph engine.
+
+Implements recursive Leiden-based hierarchical clustering that decomposes
+large communities into finer-grained thematic sub-clusters.
+"""
+import sqlite3
+from typing import List, Dict, Tuple
+
+from graph.constants import logger, IGRAPH_AVAILABLE, MEMORY_DIR
+
+
+def _get_active_profile() -> str:
+    """Get the currently active profile name from config."""
+    import json
+    config_file = MEMORY_DIR / "profiles.json"
+    if config_file.exists():
+        try:
+            with open(config_file, 'r') as f:
+                config = json.load(f)
+            return config.get('active_profile', 'default')
+        except (json.JSONDecodeError, IOError):
+            pass
+    return 'default'
+
+
+def hierarchical_cluster(db_path, get_avg_importance_fn, generate_cluster_name_fn,
+                         min_subcluster_size: int = 5, max_depth: int = 3) -> Dict[str, any]:
+    """
+    Run recursive Leiden clustering -- cluster the clusters.
+
+    Large communities (>= min_subcluster_size * 2) are recursively sub-clustered
+    to reveal finer-grained thematic structure.
+
+    Args:
+        db_path: Path to SQLite database
+        get_avg_importance_fn: Callback to compute avg importance for memory IDs
+        generate_cluster_name_fn: Callback to generate cluster name from memory IDs
+        min_subcluster_size: Minimum members to attempt sub-clustering (default 5)
+        max_depth: Maximum recursion depth (default 3)
+
+    Returns:
+        Dictionary with hierarchical clustering statistics
+    """
+    if not IGRAPH_AVAILABLE:
+        logger.warning("igraph/leidenalg not installed. Hierarchical clustering disabled. Install with: pip3 install python-igraph leidenalg")
+        return {'subclusters_created': 0, 'depth_reached': 0}
+    import igraph as ig
+    import leidenalg
+
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    active_profile = _get_active_profile()
+
+    try:
+        # Get top-level clusters for this profile that are large enough to sub-cluster
+        cursor.execute('''
+            SELECT cluster_id, COUNT(*) as cnt
+            FROM memories
+            WHERE cluster_id IS NOT NULL AND profile = ?
+            GROUP BY cluster_id
+            HAVING cnt >= ?
+        ''', (active_profile, min_subcluster_size * 2))
+        large_clusters = cursor.fetchall()
+
+        if not large_clusters:
+            logger.info("No clusters large enough for hierarchical decomposition")
+            return {'subclusters_created': 0, 'depth_reached': 0}
+
+        total_subclusters = 0
+        max_depth_reached = 0
+
+        for parent_cid, member_count in large_clusters:
+            subs, depth = _recursive_subcluster(
+                conn, cursor, parent_cid, active_profile,
+                min_subcluster_size, max_depth, current_depth=1,
+                get_avg_importance_fn=get_avg_importance_fn,
+                generate_cluster_name_fn=generate_cluster_name_fn,
+            )
+            total_subclusters += subs
+            max_depth_reached = max(max_depth_reached, depth)
+
+        conn.commit()
+        logger.info(f"Hierarchical clustering: {total_subclusters} sub-clusters, depth {max_depth_reached}")
+        return {
+            'subclusters_created': total_subclusters,
+            'depth_reached': max_depth_reached,
+            'parent_clusters_processed': len(large_clusters)
+        }
+
+    except Exception as e:
+        logger.error(f"Hierarchical clustering failed: {e}")
+        conn.rollback()
+        return {'subclusters_created': 0, 'error': str(e)}
+    finally:
+        conn.close()
+
+
+def _recursive_subcluster(conn, cursor, parent_cluster_id: int,
+                           profile: str, min_size: int, max_depth: int,
+                           current_depth: int,
+                           get_avg_importance_fn, generate_cluster_name_fn) -> Tuple[int, int]:
+    """Recursively sub-cluster a community using Leiden."""
+    if not IGRAPH_AVAILABLE:
+        return 0, current_depth - 1
+    import igraph as ig
+    import leidenalg
+
+    if current_depth > max_depth:
+        return 0, current_depth - 1
+
+    # Get memory IDs in this cluster
+    cursor.execute('''
+        SELECT id FROM memories
+        WHERE cluster_id = ? AND profile = ?
+    ''', (parent_cluster_id, profile))
+    member_ids = [row[0] for row in cursor.fetchall()]
+
+    if len(member_ids) < min_size * 2:
+        return 0, current_depth - 1
+
+    # Get edges between members of this cluster
+    placeholders = ','.join('?' * len(member_ids))
+    edges = cursor.execute(f'''
+        SELECT source_memory_id, target_memory_id, weight
+        FROM graph_edges
+        WHERE source_memory_id IN ({placeholders})
+          AND target_memory_id IN ({placeholders})
+    ''', member_ids + member_ids).fetchall()
+
+    if len(edges) < 2:
+        return 0, current_depth - 1
+
+    # Build sub-graph
+    id_to_vertex = {mid: idx for idx, mid in enumerate(member_ids)}
+    vertex_to_id = {idx: mid for mid, idx in id_to_vertex.items()}
+
+    g = ig.Graph()
+    g.add_vertices(len(member_ids))
+    edge_list, edge_weights = [], []
+    for src, tgt, w in edges:
+        if src in id_to_vertex and tgt in id_to_vertex:
+            edge_list.append((id_to_vertex[src], id_to_vertex[tgt]))
+            edge_weights.append(w)
+
+    if not edge_list:
+        return 0, current_depth - 1
+
+    g.add_edges(edge_list)
+
+    # Run Leiden with higher resolution for finer communities
+    partition = leidenalg.find_partition(
+        g, leidenalg.ModularityVertexPartition,
+        weights=edge_weights, n_iterations=100, seed=42
+    )
+
+    # Only proceed if Leiden found > 1 community (actual split)
+    non_singleton = [c for c in partition if len(c) >= 2]
+    if len(non_singleton) <= 1:
+        return 0, current_depth - 1
+
+    subclusters_created = 0
+    deepest = current_depth
+
+    # Get parent depth
+    cursor.execute('SELECT depth FROM graph_clusters WHERE id = ?', (parent_cluster_id,))
+    parent_row = cursor.fetchone()
+    parent_depth = parent_row[0] if parent_row else 0
+
+    for community in non_singleton:
+        sub_member_ids = [vertex_to_id[v] for v in community]
+
+        if len(sub_member_ids) < 2:
+            continue
+
+        avg_imp = get_avg_importance_fn(cursor, sub_member_ids)
+        cluster_name = generate_cluster_name_fn(cursor, sub_member_ids)
+
+        result = cursor.execute('''
+            INSERT INTO graph_clusters (name, member_count, avg_importance, parent_cluster_id, depth)
+            VALUES (?, ?, ?, ?, ?)
+        ''', (cluster_name, len(sub_member_ids), avg_imp, parent_cluster_id, parent_depth + 1))
+
+        sub_cluster_id = result.lastrowid
+
+        # Update memories to point to sub-cluster
+        cursor.executemany('''
+            UPDATE memories SET cluster_id = ? WHERE id = ?
+        ''', [(sub_cluster_id, mid) for mid in sub_member_ids])
+
+        subclusters_created += 1
+        logger.info(f"Sub-cluster {sub_cluster_id} under {parent_cluster_id}: "
+                    f"'{cluster_name}' ({len(sub_member_ids)} members, depth {parent_depth + 1})")
+
+        # Recurse into this sub-cluster if large enough
+        child_subs, child_depth = _recursive_subcluster(
+            conn, cursor, sub_cluster_id, profile,
+            min_size, max_depth, current_depth + 1,
+            get_avg_importance_fn=get_avg_importance_fn,
+            generate_cluster_name_fn=generate_cluster_name_fn,
+        )
+        subclusters_created += child_subs
+        deepest = max(deepest, child_depth)
+
+    return subclusters_created, deepest

package/src/graph/schema.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""Database schema management for the graph engine.
+
+Creates and maintains the graph_nodes, graph_edges, and graph_clusters
+tables, including safe schema migrations for existing databases.
+"""
+import sqlite3
+from pathlib import Path
+
+from graph.constants import logger
+
+
+def ensure_graph_tables(db_path: Path):
+    """Create graph tables if they don't exist, or recreate if schema is incomplete."""
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Check if existing tables have correct schema (not just id column)
+    for table_name, required_cols in [
+        ('graph_nodes', {'memory_id', 'entities'}),
+        ('graph_edges', {'source_memory_id', 'target_memory_id', 'weight'}),
+        ('graph_clusters', {'name', 'member_count'}),
+    ]:
+        cursor.execute(f"PRAGMA table_info({table_name})")
+        existing_cols = {row[1] for row in cursor.fetchall()}
+        if existing_cols and not required_cols.issubset(existing_cols):
+            # Table exists but has incomplete schema -- drop and recreate
+            logger.warning(f"Dropping incomplete {table_name} table (missing: {required_cols - existing_cols})")
+            cursor.execute(f'DROP TABLE IF EXISTS {table_name}')
+
+    # Graph nodes table
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS graph_nodes (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            memory_id INTEGER UNIQUE NOT NULL,
+            entities TEXT,
+            embedding_vector TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+        )
+    ''')
+
+    # Graph edges table
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS graph_edges (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source_memory_id INTEGER NOT NULL,
+            target_memory_id INTEGER NOT NULL,
+            relationship_type TEXT,
+            weight REAL DEFAULT 1.0,
+            shared_entities TEXT,
+            similarity_score REAL,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (source_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
+            FOREIGN KEY (target_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
+            UNIQUE(source_memory_id, target_memory_id)
+        )
+    ''')
+
+    # Graph clusters table
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS graph_clusters (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            description TEXT,
+            summary TEXT,
+            member_count INTEGER DEFAULT 0,
+            avg_importance REAL,
+            parent_cluster_id INTEGER,
+            depth INTEGER DEFAULT 0,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (parent_cluster_id) REFERENCES graph_clusters(id) ON DELETE SET NULL
+        )
+    ''')
+
+    # Safe column additions for existing databases
+    for col, col_type in [('summary', 'TEXT'), ('parent_cluster_id', 'INTEGER'), ('depth', 'INTEGER DEFAULT 0')]:
+        try:
+            cursor.execute(f'ALTER TABLE graph_clusters ADD COLUMN {col} {col_type}')
+        except sqlite3.OperationalError:
+            pass
+
+    # Add cluster_id to memories if not exists
+    try:
+        cursor.execute('ALTER TABLE memories ADD COLUMN cluster_id INTEGER')
+    except sqlite3.OperationalError:
+        pass  # Column already exists
+
+    # Create indexes
+    cursor.execute('CREATE INDEX IF NOT EXISTS idx_graph_source ON graph_edges(source_memory_id)')
+    cursor.execute('CREATE INDEX IF NOT EXISTS idx_graph_target ON graph_edges(target_memory_id)')
+    cursor.execute('CREATE INDEX IF NOT EXISTS idx_cluster_members ON memories(cluster_id)')
+
+    conn.commit()
+    conn.close()
+    logger.info("Graph tables initialized")