superlocalmemory 2.8.5 → 3.0.0

package/src/graph/graph_search.py

@@ -1,231 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Graph traversal and query operations.
-
-Provides graph traversal (get_related), cluster membership queries,
-and graph statistics collection for the active profile.
-"""
-import sqlite3
-import json
-from pathlib import Path
-from typing import List, Dict
-
-from graph.constants import logger, MEMORY_DIR
-
-
-def _get_active_profile() -> str:
-    """Get the currently active profile name from config."""
-    config_file = MEMORY_DIR / "profiles.json"
-    if config_file.exists():
-        try:
-            with open(config_file, 'r') as f:
-                config = json.load(f)
-            return config.get('active_profile', 'default')
-        except (json.JSONDecodeError, IOError):
-            pass
-    return 'default'
-
-
-def get_related(db_path: Path, memory_id: int, max_hops: int = 2) -> List[Dict]:
-    """
-    Get memories connected to this memory via graph edges (active profile only).
-
-    Args:
-        db_path: Path to SQLite database
-        memory_id: Source memory ID
-        max_hops: Maximum traversal depth (1 or 2)
-
-    Returns:
-        List of related memory dictionaries
-    """
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    active_profile = _get_active_profile()
-
-    try:
-        # Get 1-hop neighbors (filtered to active profile)
-        edges = cursor.execute('''
-            SELECT ge.target_memory_id, ge.relationship_type, ge.weight, ge.shared_entities
-            FROM graph_edges ge
-            JOIN memories m ON ge.target_memory_id = m.id
-            WHERE ge.source_memory_id = ? AND m.profile = ?
-            UNION
-            SELECT ge.source_memory_id, ge.relationship_type, ge.weight, ge.shared_entities
-            FROM graph_edges ge
-            JOIN memories m ON ge.source_memory_id = m.id
-            WHERE ge.target_memory_id = ? AND m.profile = ?
-        ''', (memory_id, active_profile, memory_id, active_profile)).fetchall()
-
-        results = []
-        seen_ids = {memory_id}
-
-        for target_id, rel_type, weight, shared_entities in edges:
-            if target_id in seen_ids:
-                continue
-
-            seen_ids.add(target_id)
-
-            # Get memory details
-            memory = cursor.execute('''
-                SELECT id, summary, importance, tags
-                FROM memories WHERE id = ?
-            ''', (target_id,)).fetchone()
-
-            if memory:
-                results.append({
-                    'id': memory[0],
-                    'summary': memory[1],
-                    'importance': memory[2],
-                    'tags': json.loads(memory[3]) if memory[3] else [],
-                    'relationship': rel_type,
-                    'weight': weight,
-                    'shared_entities': json.loads(shared_entities) if shared_entities else [],
-                    'hops': 1
-                })
-
-        # If max_hops == 2, get 2-hop neighbors
-        if max_hops >= 2:
-            for result in results[:]:  # Copy to avoid modification during iteration
-                second_hop = cursor.execute('''
-                    SELECT target_memory_id, relationship_type, weight
-                    FROM graph_edges
-                    WHERE source_memory_id = ?
-                    UNION
-                    SELECT source_memory_id, relationship_type, weight
-                    FROM graph_edges
-                    WHERE target_memory_id = ?
-                ''', (result['id'], result['id'])).fetchall()
-
-                for target_id, rel_type, weight in second_hop:
-                    if target_id in seen_ids:
-                        continue
-
-                    seen_ids.add(target_id)
-
-                    memory = cursor.execute('''
-                        SELECT id, summary, importance, tags
-                        FROM memories WHERE id = ?
-                    ''', (target_id,)).fetchone()
-
-                    if memory:
-                        results.append({
-                            'id': memory[0],
-                            'summary': memory[1],
-                            'importance': memory[2],
-                            'tags': json.loads(memory[3]) if memory[3] else [],
-                            'relationship': rel_type,
-                            'weight': weight,
-                            'shared_entities': [],
-                            'hops': 2
-                        })
-
-        # Sort by weight (strongest connections first)
-        results.sort(key=lambda x: (-x['hops'], -x['weight']))
-
-        return results
-
-    finally:
-        conn.close()
-
-
-def get_cluster_members(db_path: Path, cluster_id: int) -> List[Dict]:
-    """
-    Get all memories in a cluster (filtered by active profile).
-
-    Args:
-        db_path: Path to SQLite database
-        cluster_id: Cluster ID
-
-    Returns:
-        List of memory dictionaries
-    """
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    active_profile = _get_active_profile()
-
-    try:
-        memories = cursor.execute('''
-            SELECT id, summary, importance, tags, created_at
-            FROM memories
-            WHERE cluster_id = ? AND profile = ?
-            ORDER BY importance DESC
-        ''', (cluster_id, active_profile)).fetchall()
-
-        return [
-            {
-                'id': m[0],
-                'summary': m[1],
-                'importance': m[2],
-                'tags': json.loads(m[3]) if m[3] else [],
-                'created_at': m[4]
-            }
-            for m in memories
-        ]
-
-    finally:
-        conn.close()
-
-
-def get_stats(db_path: Path) -> Dict[str, any]:
-    """Get graph statistics for the active profile."""
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    active_profile = _get_active_profile()
-
-    try:
-        # Count nodes for active profile's memories
-        nodes = cursor.execute('''
-            SELECT COUNT(*) FROM graph_nodes
-            WHERE memory_id IN (SELECT id FROM memories WHERE profile = ?)
-        ''', (active_profile,)).fetchone()[0]
-
-        # Count edges where at least one end is in active profile
-        edges = cursor.execute('''
-            SELECT COUNT(*) FROM graph_edges
-            WHERE source_memory_id IN (SELECT id FROM memories WHERE profile = ?)
-        ''', (active_profile,)).fetchone()[0]
-
-        # Clusters that have members in active profile
-        clusters = cursor.execute('''
-            SELECT COUNT(DISTINCT cluster_id) FROM memories
-            WHERE cluster_id IS NOT NULL AND profile = ?
-        ''', (active_profile,)).fetchone()[0]
-
-        # Cluster breakdown for active profile (including hierarchy)
-        cluster_info = cursor.execute('''
-            SELECT gc.name, gc.member_count, gc.avg_importance,
-                   gc.summary, gc.parent_cluster_id, gc.depth
-            FROM graph_clusters gc
-            WHERE gc.id IN (
-                SELECT DISTINCT cluster_id FROM memories
-                WHERE cluster_id IS NOT NULL AND profile = ?
-            )
-            ORDER BY gc.depth ASC, gc.member_count DESC
-            LIMIT 20
-        ''', (active_profile,)).fetchall()
-
-        # Count hierarchical depth
-        max_depth = max((c[5] or 0 for c in cluster_info), default=0) if cluster_info else 0
-
-        return {
-            'profile': active_profile,
-            'nodes': nodes,
-            'edges': edges,
-            'clusters': clusters,
-            'max_depth': max_depth,
-            'top_clusters': [
-                {
-                    'name': c[0],
-                    'members': c[1],
-                    'avg_importance': round(c[2], 1) if c[2] else 5.0,
-                    'summary': c[3],
-                    'parent_cluster_id': c[4],
-                    'depth': c[5] or 0
-                }
-                for c in cluster_info
-            ]
-        }
-
-    finally:
-        conn.close()

package/src/graph/hierarchical.py

@@ -1,207 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Hierarchical sub-clustering for the graph engine.
-
-Implements recursive Leiden-based hierarchical clustering that decomposes
-large communities into finer-grained thematic sub-clusters.
-"""
-import sqlite3
-from typing import List, Dict, Tuple
-
-from graph.constants import logger, IGRAPH_AVAILABLE, MEMORY_DIR
-
-
-def _get_active_profile() -> str:
-    """Get the currently active profile name from config."""
-    import json
-    config_file = MEMORY_DIR / "profiles.json"
-    if config_file.exists():
-        try:
-            with open(config_file, 'r') as f:
-                config = json.load(f)
-            return config.get('active_profile', 'default')
-        except (json.JSONDecodeError, IOError):
-            pass
-    return 'default'
-
-
-def hierarchical_cluster(db_path, get_avg_importance_fn, generate_cluster_name_fn,
-                         min_subcluster_size: int = 5, max_depth: int = 3) -> Dict[str, any]:
-    """
-    Run recursive Leiden clustering -- cluster the clusters.
-
-    Large communities (>= min_subcluster_size * 2) are recursively sub-clustered
-    to reveal finer-grained thematic structure.
-
-    Args:
-        db_path: Path to SQLite database
-        get_avg_importance_fn: Callback to compute avg importance for memory IDs
-        generate_cluster_name_fn: Callback to generate cluster name from memory IDs
-        min_subcluster_size: Minimum members to attempt sub-clustering (default 5)
-        max_depth: Maximum recursion depth (default 3)
-
-    Returns:
-        Dictionary with hierarchical clustering statistics
-    """
-    if not IGRAPH_AVAILABLE:
-        logger.warning("igraph/leidenalg not installed. Hierarchical clustering disabled. Install with: pip3 install python-igraph leidenalg")
-        return {'subclusters_created': 0, 'depth_reached': 0}
-    import igraph as ig
-    import leidenalg
-
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    active_profile = _get_active_profile()
-
-    try:
-        # Get top-level clusters for this profile that are large enough to sub-cluster
-        cursor.execute('''
-            SELECT cluster_id, COUNT(*) as cnt
-            FROM memories
-            WHERE cluster_id IS NOT NULL AND profile = ?
-            GROUP BY cluster_id
-            HAVING cnt >= ?
-        ''', (active_profile, min_subcluster_size * 2))
-        large_clusters = cursor.fetchall()
-
-        if not large_clusters:
-            logger.info("No clusters large enough for hierarchical decomposition")
-            return {'subclusters_created': 0, 'depth_reached': 0}
-
-        total_subclusters = 0
-        max_depth_reached = 0
-
-        for parent_cid, member_count in large_clusters:
-            subs, depth = _recursive_subcluster(
-                conn, cursor, parent_cid, active_profile,
-                min_subcluster_size, max_depth, current_depth=1,
-                get_avg_importance_fn=get_avg_importance_fn,
-                generate_cluster_name_fn=generate_cluster_name_fn,
-            )
-            total_subclusters += subs
-            max_depth_reached = max(max_depth_reached, depth)
-
-        conn.commit()
-        logger.info(f"Hierarchical clustering: {total_subclusters} sub-clusters, depth {max_depth_reached}")
-        return {
-            'subclusters_created': total_subclusters,
-            'depth_reached': max_depth_reached,
-            'parent_clusters_processed': len(large_clusters)
-        }
-
-    except Exception as e:
-        logger.error(f"Hierarchical clustering failed: {e}")
-        conn.rollback()
-        return {'subclusters_created': 0, 'error': str(e)}
-    finally:
-        conn.close()
-
-
-def _recursive_subcluster(conn, cursor, parent_cluster_id: int,
-                           profile: str, min_size: int, max_depth: int,
-                           current_depth: int,
-                           get_avg_importance_fn, generate_cluster_name_fn) -> Tuple[int, int]:
-    """Recursively sub-cluster a community using Leiden."""
-    if not IGRAPH_AVAILABLE:
-        return 0, current_depth - 1
-    import igraph as ig
-    import leidenalg
-
-    if current_depth > max_depth:
-        return 0, current_depth - 1
-
-    # Get memory IDs in this cluster
-    cursor.execute('''
-        SELECT id FROM memories
-        WHERE cluster_id = ? AND profile = ?
-    ''', (parent_cluster_id, profile))
-    member_ids = [row[0] for row in cursor.fetchall()]
-
-    if len(member_ids) < min_size * 2:
-        return 0, current_depth - 1
-
-    # Get edges between members of this cluster
-    placeholders = ','.join('?' * len(member_ids))
-    edges = cursor.execute(f'''
-        SELECT source_memory_id, target_memory_id, weight
-        FROM graph_edges
-        WHERE source_memory_id IN ({placeholders})
-          AND target_memory_id IN ({placeholders})
-    ''', member_ids + member_ids).fetchall()
-
-    if len(edges) < 2:
-        return 0, current_depth - 1
-
-    # Build sub-graph
-    id_to_vertex = {mid: idx for idx, mid in enumerate(member_ids)}
-    vertex_to_id = {idx: mid for mid, idx in id_to_vertex.items()}
-
-    g = ig.Graph()
-    g.add_vertices(len(member_ids))
-    edge_list, edge_weights = [], []
-    for src, tgt, w in edges:
-        if src in id_to_vertex and tgt in id_to_vertex:
-            edge_list.append((id_to_vertex[src], id_to_vertex[tgt]))
-            edge_weights.append(w)
-
-    if not edge_list:
-        return 0, current_depth - 1
-
-    g.add_edges(edge_list)
-
-    # Run Leiden with higher resolution for finer communities
-    partition = leidenalg.find_partition(
-        g, leidenalg.ModularityVertexPartition,
-        weights=edge_weights, n_iterations=100, seed=42
-    )
-
-    # Only proceed if Leiden found > 1 community (actual split)
-    non_singleton = [c for c in partition if len(c) >= 2]
-    if len(non_singleton) <= 1:
-        return 0, current_depth - 1
-
-    subclusters_created = 0
-    deepest = current_depth
-
-    # Get parent depth
-    cursor.execute('SELECT depth FROM graph_clusters WHERE id = ?', (parent_cluster_id,))
-    parent_row = cursor.fetchone()
-    parent_depth = parent_row[0] if parent_row else 0
-
-    for community in non_singleton:
-        sub_member_ids = [vertex_to_id[v] for v in community]
-
-        if len(sub_member_ids) < 2:
-            continue
-
-        avg_imp = get_avg_importance_fn(cursor, sub_member_ids)
-        cluster_name = generate_cluster_name_fn(cursor, sub_member_ids)
-
-        result = cursor.execute('''
-            INSERT INTO graph_clusters (name, member_count, avg_importance, parent_cluster_id, depth)
-            VALUES (?, ?, ?, ?, ?)
-        ''', (cluster_name, len(sub_member_ids), avg_imp, parent_cluster_id, parent_depth + 1))
-
-        sub_cluster_id = result.lastrowid
-
-        # Update memories to point to sub-cluster
-        cursor.executemany('''
-            UPDATE memories SET cluster_id = ? WHERE id = ?
-        ''', [(sub_cluster_id, mid) for mid in sub_member_ids])
-
-        subclusters_created += 1
-        logger.info(f"Sub-cluster {sub_cluster_id} under {parent_cluster_id}: "
-                    f"'{cluster_name}' ({len(sub_member_ids)} members, depth {parent_depth + 1})")
-
-        # Recurse into this sub-cluster if large enough
-        child_subs, child_depth = _recursive_subcluster(
-            conn, cursor, sub_cluster_id, profile,
-            min_size, max_depth, current_depth + 1,
-            get_avg_importance_fn=get_avg_importance_fn,
-            generate_cluster_name_fn=generate_cluster_name_fn,
-        )
-        subclusters_created += child_subs
-        deepest = max(deepest, child_depth)
-
-    return subclusters_created, deepest

package/src/graph/schema.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""Database schema management for the graph engine.
-
-Creates and maintains the graph_nodes, graph_edges, and graph_clusters
-tables, including safe schema migrations for existing databases.
-"""
-import sqlite3
-from pathlib import Path
-
-from graph.constants import logger
-
-
-def ensure_graph_tables(db_path: Path):
-    """Create graph tables if they don't exist, or recreate if schema is incomplete."""
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-
-    # Check if existing tables have correct schema (not just id column)
-    for table_name, required_cols in [
-        ('graph_nodes', {'memory_id', 'entities'}),
-        ('graph_edges', {'source_memory_id', 'target_memory_id', 'weight'}),
-        ('graph_clusters', {'name', 'member_count'}),
-    ]:
-        cursor.execute(f"PRAGMA table_info({table_name})")
-        existing_cols = {row[1] for row in cursor.fetchall()}
-        if existing_cols and not required_cols.issubset(existing_cols):
-            # Table exists but has incomplete schema -- drop and recreate
-            logger.warning(f"Dropping incomplete {table_name} table (missing: {required_cols - existing_cols})")
-            cursor.execute(f'DROP TABLE IF EXISTS {table_name}')
-
-    # Graph nodes table
-    cursor.execute('''
-        CREATE TABLE IF NOT EXISTS graph_nodes (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            memory_id INTEGER UNIQUE NOT NULL,
-            entities TEXT,
-            embedding_vector TEXT,
-            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
-        )
-    ''')
-
-    # Graph edges table
-    cursor.execute('''
-        CREATE TABLE IF NOT EXISTS graph_edges (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            source_memory_id INTEGER NOT NULL,
-            target_memory_id INTEGER NOT NULL,
-            relationship_type TEXT,
-            weight REAL DEFAULT 1.0,
-            shared_entities TEXT,
-            similarity_score REAL,
-            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (source_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
-            FOREIGN KEY (target_memory_id) REFERENCES memories(id) ON DELETE CASCADE,
-            UNIQUE(source_memory_id, target_memory_id)
-        )
-    ''')
-
-    # Graph clusters table
-    cursor.execute('''
-        CREATE TABLE IF NOT EXISTS graph_clusters (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT NOT NULL,
-            description TEXT,
-            summary TEXT,
-            member_count INTEGER DEFAULT 0,
-            avg_importance REAL,
-            parent_cluster_id INTEGER,
-            depth INTEGER DEFAULT 0,
-            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (parent_cluster_id) REFERENCES graph_clusters(id) ON DELETE SET NULL
-        )
-    ''')
-
-    # Safe column additions for existing databases
-    for col, col_type in [('summary', 'TEXT'), ('parent_cluster_id', 'INTEGER'), ('depth', 'INTEGER DEFAULT 0')]:
-        try:
-            cursor.execute(f'ALTER TABLE graph_clusters ADD COLUMN {col} {col_type}')
-        except sqlite3.OperationalError:
-            pass
-
-    # Add cluster_id to memories if not exists
-    try:
-        cursor.execute('ALTER TABLE memories ADD COLUMN cluster_id INTEGER')
-    except sqlite3.OperationalError:
-        pass  # Column already exists
-
-    # Create indexes
-    cursor.execute('CREATE INDEX IF NOT EXISTS idx_graph_source ON graph_edges(source_memory_id)')
-    cursor.execute('CREATE INDEX IF NOT EXISTS idx_graph_target ON graph_edges(target_memory_id)')
-    cursor.execute('CREATE INDEX IF NOT EXISTS idx_cluster_members ON memories(cluster_id)')
-
-    conn.commit()
-    conn.close()
-    logger.info("Graph tables initialized")

package/src/graph_engine.py

@@ -1,52 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""GraphEngine - Knowledge Graph Clustering for SuperLocalMemory V2
-
-BACKWARD-COMPATIBILITY SHIM
-----------------------------
-This file re-exports every public symbol from the ``graph`` package so that
-existing code using ``from graph_engine import GraphEngine`` (or any other
-name) continues to work without modification.
-
-The actual implementation now lives in:
-    src/graph/constants.py        - Shared imports, constants, logger
-    src/graph/entity_extractor.py - EntityExtractor, ClusterNamer
-    src/graph/edge_builder.py     - EdgeBuilder
-    src/graph/cluster_builder.py  - ClusterBuilder
-    src/graph/graph_core.py       - GraphEngine, main()
-"""
-# Re-export everything from the graph package
-from graph import (
-    # Constants
-    MAX_MEMORIES_FOR_GRAPH,
-    SKLEARN_AVAILABLE,
-    IGRAPH_AVAILABLE,
-    MEMORY_DIR,
-    DB_PATH,
-    # Classes
-    EntityExtractor,
-    ClusterNamer,
-    EdgeBuilder,
-    ClusterBuilder,
-    GraphEngine,
-    # Functions
-    main,
-)
-
-__all__ = [
-    "MAX_MEMORIES_FOR_GRAPH",
-    "SKLEARN_AVAILABLE",
-    "IGRAPH_AVAILABLE",
-    "MEMORY_DIR",
-    "DB_PATH",
-    "EntityExtractor",
-    "ClusterNamer",
-    "EdgeBuilder",
-    "ClusterBuilder",
-    "GraphEngine",
-    "main",
-]
-
-if __name__ == '__main__':
-    main()