superlocalmemory 2.8.2 → 2.8.5

package/src/learning/synthetic_bootstrap.py

@@ -36,9 +36,6 @@ Research Backing:
 
 import hashlib
 import logging
-import re
-import sqlite3
-from collections import Counter
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Set
@@ -59,79 +56,27 @@ except ImportError:
     HAS_NUMPY = False
 
 from .feature_extractor import FeatureExtractor, FEATURE_NAMES, NUM_FEATURES
+from .bootstrap import (
+    MEMORY_DB_PATH,
+    MODELS_DIR,
+    MODEL_PATH,
+    MIN_MEMORIES_FOR_BOOTSTRAP,
+    BOOTSTRAP_CONFIG,
+    BOOTSTRAP_PARAMS,
+    extract_keywords,
+    get_memory_count,
+    get_memories_by_access,
+    get_memories_by_importance,
+    get_recent_memories,
+    get_learned_patterns,
+    search_memories,
+    find_negative_memories,
+    diverse_sample,
+    count_sources,
+)
 
 logger = logging.getLogger("superlocalmemory.learning.synthetic_bootstrap")
 
-# ============================================================================
-# Constants
-# ============================================================================
-
-MEMORY_DB_PATH = Path.home() / ".claude-memory" / "memory.db"
-MODELS_DIR = Path.home() / ".claude-memory" / "models"
-MODEL_PATH = MODELS_DIR / "ranker.txt"
-
-# Minimum memories needed before bootstrap makes sense
-MIN_MEMORIES_FOR_BOOTSTRAP = 50
-
-# Tiered config — bootstrap model complexity scales with data size
-BOOTSTRAP_CONFIG = {
-    'small': {
-        'min_memories': 50,
-        'max_memories': 499,
-        'target_samples': 200,
-        'n_estimators': 30,
-        'max_depth': 3,
-    },
-    'medium': {
-        'min_memories': 500,
-        'max_memories': 4999,
-        'target_samples': 1000,
-        'n_estimators': 50,
-        'max_depth': 4,
-    },
-    'large': {
-        'min_memories': 5000,
-        'max_memories': float('inf'),
-        'target_samples': 2000,
-        'n_estimators': 100,
-        'max_depth': 6,
-    },
-}
-
-# LightGBM bootstrap parameters — MORE aggressive regularization than
-# real training because synthetic data has systematic biases
-BOOTSTRAP_PARAMS = {
-    'objective': 'lambdarank',
-    'metric': 'ndcg',
-    'ndcg_eval_at': [5, 10],
-    'learning_rate': 0.1,
-    'num_leaves': 8,
-    'max_depth': 3,
-    'min_child_samples': 5,
-    'subsample': 0.7,
-    'reg_alpha': 0.5,
-    'reg_lambda': 2.0,
-    'boosting_type': 'dart',
-    'verbose': -1,
-}
-
-# English stopwords for keyword extraction (no external deps)
-_STOPWORDS = frozenset({
-    'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
-    'of', 'with', 'by', 'from', 'is', 'it', 'this', 'that', 'was', 'are',
-    'be', 'has', 'have', 'had', 'do', 'does', 'did', 'will', 'would',
-    'could', 'should', 'may', 'might', 'can', 'not', 'no', 'if', 'then',
-    'so', 'as', 'up', 'out', 'about', 'into', 'over', 'after', 'before',
-    'when', 'where', 'how', 'what', 'which', 'who', 'whom', 'why',
-    'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other',
-    'some', 'such', 'than', 'too', 'very', 'just', 'also', 'now',
-    'here', 'there', 'use', 'used', 'using', 'make', 'made',
-    'need', 'needed', 'get', 'got', 'set', 'new', 'old', 'one', 'two',
-})
-
-# Minimum word length for keyword extraction
-_MIN_KEYWORD_LENGTH = 3
-
 
 class SyntheticBootstrapper:
     """
@@ -232,18 +177,7 @@ class SyntheticBootstrapper:
 
     def _get_memory_count(self) -> int:
         """Count total memories in memory.db."""
-        if not self._memory_db.exists():
-            return 0
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            cursor = conn.cursor()
-            cursor.execute('SELECT COUNT(*) FROM memories')
-            count = cursor.fetchone()[0]
-            conn.close()
-            return count
-        except Exception as e:
-            logger.warning("Failed to count memories: %s", e)
-            return 0
+        return get_memory_count(self._memory_db)
 
     # ========================================================================
     # Synthetic Data Generation
@@ -699,26 +633,7 @@ class SyntheticBootstrapper:
 
         These are memories the user keeps coming back to — strong positive signal.
         """
-        if not self._memory_db.exists():
-            return []
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, content, summary, project_name, tags,
-                       category, importance, created_at, access_count
-                FROM memories
-                WHERE access_count >= ?
-                ORDER BY access_count DESC
-                LIMIT 100
-            ''', (min_access,))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.warning("Failed to fetch high-access memories: %s", e)
-            return []
+        return get_memories_by_access(self._memory_db, min_access)
 
     def _get_memories_by_importance(self, min_importance: int = 8) -> List[dict]:
         """
@@ -726,48 +641,11 @@ class SyntheticBootstrapper:
 
         High importance = user explicitly rated these as valuable.
         """
-        if not self._memory_db.exists():
-            return []
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, content, summary, project_name, tags,
-                       category, importance, created_at, access_count
-                FROM memories
-                WHERE importance >= ?
-                ORDER BY importance DESC
-                LIMIT 100
-            ''', (min_importance,))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.warning("Failed to fetch high-importance memories: %s", e)
-            return []
+        return get_memories_by_importance(self._memory_db, min_importance)
 
     def _get_recent_memories(self, limit: int = 30) -> List[dict]:
         """Fetch the N most recently created memories."""
-        if not self._memory_db.exists():
-            return []
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, content, summary, project_name, tags,
-                       category, importance, created_at, access_count
-                FROM memories
-                ORDER BY created_at DESC
-                LIMIT ?
-            ''', (limit,))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.warning("Failed to fetch recent memories: %s", e)
-            return []
+        return get_recent_memories(self._memory_db, limit)
 
     def _get_learned_patterns(
         self,
@@ -782,36 +660,7 @@ class SyntheticBootstrapper:
         Returns empty list if identity_patterns table doesn't exist
         (backward compatible with pre-v2.3 databases).
         """
-        if not self._memory_db.exists():
-            return []
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            # Check if table exists (backward compatibility)
-            cursor.execute('''
-                SELECT name FROM sqlite_master
-                WHERE type='table' AND name='identity_patterns'
-            ''')
-            if cursor.fetchone() is None:
-                conn.close()
-                return []
-
-            cursor.execute('''
-                SELECT id, pattern_type, key, value, confidence,
-                       evidence_count, category
-                FROM identity_patterns
-                WHERE confidence >= ?
-                ORDER BY confidence DESC
-                LIMIT 50
-            ''', (min_confidence,))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.warning("Failed to fetch learned patterns: %s", e)
-            return []
+        return get_learned_patterns(self._memory_db, min_confidence)
 
     def _search_memories(self, query: str, limit: int = 20) -> List[dict]:
         """
@@ -820,38 +669,7 @@ class SyntheticBootstrapper:
         Used to find memories matching synthetic query terms.
         This is a lightweight search — no TF-IDF, no HNSW, just FTS5.
         """
-        if not self._memory_db.exists():
-            return []
-        if not query or not query.strip():
-            return []
-
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            # Clean query for FTS5 (same approach as memory_store_v2.search)
-            fts_tokens = re.findall(r'\w+', query)
-            if not fts_tokens:
-                conn.close()
-                return []
-            fts_query = ' OR '.join(fts_tokens)
-
-            cursor.execute('''
-                SELECT m.id, m.content, m.summary, m.project_name, m.tags,
-                       m.category, m.importance, m.created_at, m.access_count
-                FROM memories m
-                JOIN memories_fts fts ON m.id = fts.rowid
-                WHERE memories_fts MATCH ?
-                ORDER BY rank
-                LIMIT ?
-            ''', (fts_query, limit))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.debug("FTS5 search failed (may not exist yet): %s", e)
-            return []
+        return search_memories(self._memory_db, query, limit)
 
     def _find_negative_memories(
         self,
@@ -865,51 +683,7 @@ class SyntheticBootstrapper:
         Simple heuristic: pick memories from a different category or project.
         Falls back to random sample if no structured differences available.
         """
-        if not self._memory_db.exists():
-            return []
-        exclude_ids = exclude_ids or set()
-
-        try:
-            conn = sqlite3.connect(str(self._memory_db), timeout=5)
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            anchor_project = anchor_memory.get('project_name', '')
-            anchor_category = anchor_memory.get('category', '')
-
-            # Try to find memories from different project or category
-            conditions = []
-            params: list = []
-
-            if anchor_project:
-                conditions.append('project_name != ?')
-                params.append(anchor_project)
-            if anchor_category:
-                conditions.append('category != ?')
-                params.append(anchor_category)
-
-            # Exclude specified IDs
-            if exclude_ids:
-                placeholders = ','.join('?' for _ in exclude_ids)
-                conditions.append(f'id NOT IN ({placeholders})')
-                params.extend(exclude_ids)
-
-            where_clause = ' AND '.join(conditions) if conditions else '1=1'
-
-            cursor.execute(f'''
-                SELECT id, content, summary, project_name, tags,
-                       category, importance, created_at, access_count
-                FROM memories
-                WHERE {where_clause}
-                ORDER BY RANDOM()
-                LIMIT ?
-            ''', (*params, limit))
-            results = [dict(row) for row in cursor.fetchall()]
-            conn.close()
-            return results
-        except Exception as e:
-            logger.debug("Failed to find negative memories: %s", e)
-            return []
+        return find_negative_memories(self._memory_db, anchor_memory, exclude_ids, limit)
 
     # ========================================================================
     # Text Processing
@@ -926,24 +700,7 @@ class SyntheticBootstrapper:
 
         No external NLP dependencies — just regex + counter.
         """
-        if not content:
-            return []
-
-        # Tokenize: extract alphanumeric words
-        words = re.findall(r'[a-zA-Z][a-zA-Z0-9_.-]*[a-zA-Z0-9]|[a-zA-Z]', content.lower())
-
-        # Filter stopwords and short words
-        meaningful = [
-            w for w in words
-            if w not in _STOPWORDS and len(w) >= _MIN_KEYWORD_LENGTH
-        ]
-
-        if not meaningful:
-            return []
-
-        # Count and return top N
-        counter = Counter(meaningful)
-        return [word for word, _count in counter.most_common(top_n)]
+        return extract_keywords(content, top_n)
 
     # ========================================================================
     # Utility
@@ -960,50 +717,11 @@ class SyntheticBootstrapper:
         Takes proportional samples from each source strategy to ensure
         the training data isn't dominated by one strategy.
         """
-        if len(records) <= target:
-            return records
-
-        # Group by source
-        by_source: Dict[str, List[dict]] = {}
-        for r in records:
-            src = r.get('source', 'unknown')
-            if src not in by_source:
-                by_source[src] = []
-            by_source[src].append(r)
-
-        # Proportional allocation
-        n_sources = len(by_source)
-        if n_sources == 0:
-            return records[:target]
-
-        per_source = max(1, target // n_sources)
-        sampled = []
-
-        for source, source_records in by_source.items():
-            # Take up to per_source from each, or all if fewer
-            take = min(len(source_records), per_source)
-            sampled.extend(source_records[:take])
-
-        # If under target, fill from remaining
-        if len(sampled) < target:
-            used_ids = {(r['query_hash'], r['memory_id']) for r in sampled}
-            for r in records:
-                if len(sampled) >= target:
-                    break
-                key = (r['query_hash'], r['memory_id'])
-                if key not in used_ids:
-                    sampled.append(r)
-                    used_ids.add(key)
-
-        return sampled[:target]
+        return diverse_sample(records, target)
 
     def _count_sources(self, records: List[dict]) -> Dict[str, int]:
         """Count records by source strategy."""
-        counts: Dict[str, int] = {}
-        for r in records:
-            src = r.get('source', 'unknown')
-            counts[src] = counts.get(src, 0) + 1
-        return counts
+        return count_sources(records)
 
 
 # ============================================================================

package/src/memory/__init__.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Memory package - Constants, schema, and utilities for MemoryStoreV2.
+
+This package contains extracted utilities from memory_store_v2.py to keep
+the main class focused and under the 800-line target.
+"""
+
+from .constants import (
+    MEMORY_DIR,
+    DB_PATH,
+    VECTORS_PATH,
+    MAX_CONTENT_SIZE,
+    MAX_SUMMARY_SIZE,
+    MAX_TAG_LENGTH,
+    MAX_TAGS,
+    CREATOR_METADATA,
+)
+
+from .helpers import format_content
+from .cli import run_cli
+
+__all__ = [
+    'MEMORY_DIR',
+    'DB_PATH',
+    'VECTORS_PATH',
+    'MAX_CONTENT_SIZE',
+    'MAX_SUMMARY_SIZE',
+    'MAX_TAG_LENGTH',
+    'MAX_TAGS',
+    'CREATOR_METADATA',
+    'format_content',
+    'run_cli',
+]

package/src/memory/cli.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Command-line interface for MemoryStoreV2.
+
+This module contains the CLI implementation extracted from memory_store_v2.py
+to reduce file size and improve maintainability.
+"""
+
+import sys
+import json
+from .helpers import format_content
+
+
+def run_cli():
+    """Main CLI entry point for MemoryStoreV2."""
+    # Import here to avoid circular dependency
+    from memory_store_v2 import MemoryStoreV2
+
+    store = MemoryStoreV2()
+
+    if len(sys.argv) < 2:
+        print("MemoryStore V2 CLI")
+        print("\nV1 Compatible Commands:")
+        print("  python memory_store_v2.py add <content> [--project <path>] [--tags tag1,tag2]")
+        print("  python memory_store_v2.py search <query> [--full]")
+        print("  python memory_store_v2.py list [limit] [--full]")
+        print("  python memory_store_v2.py get <id>")
+        print("  python memory_store_v2.py recent [limit] [--full]")
+        print("  python memory_store_v2.py stats")
+        print("  python memory_store_v2.py context <query>")
+        print("  python memory_store_v2.py delete <id>")
+        print("\nV2 Extensions:")
+        print("  python memory_store_v2.py tree [parent_id]")
+        print("  python memory_store_v2.py cluster <cluster_id> [--full]")
+        print("\nOptions:")
+        print("  --full    Show complete content (default: smart truncation at 5000 chars)")
+        sys.exit(0)
+
+    command = sys.argv[1]
+
+    if command == "tree":
+        parent_id = int(sys.argv[2]) if len(sys.argv) > 2 else None
+        results = store.get_tree(parent_id)
+
+        if not results:
+            print("No memories in tree.")
+        else:
+            for r in results:
+                indent = "  " * r['depth']
+                print(f"{indent}[{r['id']}] {r['content'][:50]}...")
+                if r.get('category'):
+                    print(f"{indent}    Category: {r['category']}")
+
+    elif command == "cluster" and len(sys.argv) >= 3:
+        cluster_id = int(sys.argv[2])
+        show_full = '--full' in sys.argv
+        results = store.get_by_cluster(cluster_id)
+
+        if not results:
+            print(f"No memories in cluster {cluster_id}.")
+        else:
+            print(f"Cluster {cluster_id} - {len(results)} memories:")
+            for r in results:
+                print(f"\n[{r['id']}] Importance: {r['importance']}")
+                print(f"  {format_content(r['content'], full=show_full)}")
+
+    elif command == "stats":
+        stats = store.get_stats()
+        print(json.dumps(stats, indent=2))
+
+    elif command == "add":
+        # Parse content and options
+        if len(sys.argv) < 3:
+            print("Error: Content required")
+            print("Usage: python memory_store_v2.py add <content> [--project <path>] [--tags tag1,tag2]")
+            sys.exit(1)
+
+        content = sys.argv[2]
+        project_path = None
+        tags = []
+
+        i = 3
+        while i < len(sys.argv):
+            if sys.argv[i] == '--project' and i + 1 < len(sys.argv):
+                project_path = sys.argv[i + 1]
+                i += 2
+            elif sys.argv[i] == '--tags' and i + 1 < len(sys.argv):
+                tags = [t.strip() for t in sys.argv[i + 1].split(',')]
+                i += 2
+            else:
+                i += 1
+
+        mem_id = store.add_memory(content, project_path=project_path, tags=tags)
+        print(f"Memory added with ID: {mem_id}")
+
+    elif command == "search":
+        if len(sys.argv) < 3:
+            print("Error: Search query required")
+            print("Usage: python memory_store_v2.py search <query> [--full]")
+            sys.exit(1)
+
+        query = sys.argv[2]
+        show_full = '--full' in sys.argv
+        results = store.search(query, limit=5)
+
+        if not results:
+            print("No results found.")
+        else:
+            for r in results:
+                print(f"\n[{r['id']}] Score: {r['score']:.2f}")
+                if r.get('project_name'):
+                    print(f"Project: {r['project_name']}")
+                if r.get('tags'):
+                    print(f"Tags: {', '.join(r['tags'])}")
+                print(f"Content: {format_content(r['content'], full=show_full)}")
+                print(f"Created: {r['created_at']}")
+
+    elif command == "recent":
+        show_full = '--full' in sys.argv
+        # Parse limit (skip --full flag)
+        limit = 10
+        for i, arg in enumerate(sys.argv[2:], start=2):
+            if arg != '--full' and arg.isdigit():
+                limit = int(arg)
+                break
+
+        results = store.get_recent(limit)
+
+        if not results:
+            print("No memories found.")
+        else:
+            for r in results:
+                print(f"\n[{r['id']}] {r['created_at']}")
+                if r.get('project_name'):
+                    print(f"Project: {r['project_name']}")
+                if r.get('tags'):
+                    print(f"Tags: {', '.join(r['tags'])}")
+                print(f"Content: {format_content(r['content'], full=show_full)}")
+
+    elif command == "list":
+        show_full = '--full' in sys.argv
+        # Parse limit (skip --full flag)
+        limit = 10
+        for i, arg in enumerate(sys.argv[2:], start=2):
+            if arg != '--full' and arg.isdigit():
+                limit = int(arg)
+                break
+
+        results = store.get_recent(limit)
+
+        if not results:
+            print("No memories found.")
+        else:
+            for r in results:
+                print(f"[{r['id']}] {format_content(r['content'], full=show_full)}")
+
+    elif command == "get":
+        if len(sys.argv) < 3:
+            print("Error: Memory ID required")
+            print("Usage: python memory_store_v2.py get <id>")
+            sys.exit(1)
+
+        mem_id = int(sys.argv[2])
+        memory = store.get_by_id(mem_id)
+
+        if not memory:
+            print(f"Memory {mem_id} not found.")
+        else:
+            print(f"\nID: {memory['id']}")
+            print(f"Content: {memory['content']}")
+            if memory.get('summary'):
+                print(f"Summary: {memory['summary']}")
+            if memory.get('project_name'):
+                print(f"Project: {memory['project_name']}")
+            if memory.get('tags'):
+                print(f"Tags: {', '.join(memory['tags'])}")
+            print(f"Created: {memory['created_at']}")
+            print(f"Importance: {memory['importance']}")
+            print(f"Access Count: {memory['access_count']}")
+
+    elif command == "context":
+        if len(sys.argv) < 3:
+            print("Error: Query required")
+            print("Usage: python memory_store_v2.py context <query>")
+            sys.exit(1)
+
+        query = sys.argv[2]
+        context = store.export_for_context(query)
+        print(context)
+
+    elif command == "delete":
+        if len(sys.argv) < 3:
+            print("Error: Memory ID required")
+            print("Usage: python memory_store_v2.py delete <id>")
+            sys.exit(1)
+
+        mem_id = int(sys.argv[2])
+        store.delete_memory(mem_id)
+        print(f"Memory {mem_id} deleted.")
+
+    else:
+        print(f"Unknown command: {command}")
+        print("Run without arguments to see available commands.")

package/src/memory/constants.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Constants and configuration values for MemoryStoreV2.
+
+This module contains all module-level constants extracted from memory_store_v2.py
+to reduce file size and improve maintainability.
+"""
+
+from pathlib import Path
+
+# Database paths
+MEMORY_DIR = Path.home() / ".claude-memory"
+DB_PATH = MEMORY_DIR / "memory.db"
+VECTORS_PATH = MEMORY_DIR / "vectors"
+
+# Security: Input validation limits
+MAX_CONTENT_SIZE = 1_000_000    # 1MB max content
+MAX_SUMMARY_SIZE = 10_000       # 10KB max summary
+MAX_TAG_LENGTH = 50             # 50 chars per tag
+MAX_TAGS = 20                   # 20 tags max
+
+# Creator Attribution Metadata (REQUIRED by MIT License)
+# This data is embedded in the database creator_metadata table
+CREATOR_METADATA = {
+    'creator_name': 'Varun Pratap Bhardwaj',
+    'creator_role': 'Solution Architect & Original Creator',
+    'creator_github': 'varun369',
+    'project_name': 'SuperLocalMemory V2',
+    'project_url': 'https://github.com/varun369/SuperLocalMemoryV2',
+    'license': 'MIT',
+    'attribution_required': 'yes',
+    'version': '2.5.0',
+    'architecture_date': '2026-01-15',
+    'release_date': '2026-02-07',
+    'signature': 'VBPB-SLM-V2-2026-ARCHITECT',
+    'verification_hash': 'sha256:c9f3d1a8b5e2f4c6d8a9b3e7f1c4d6a8b9c3e7f2d5a8c1b4e6f9d2a7c5b8e1'
+}