superlocalmemory 2.8.2 → 2.8.5

package/src/learning/bootstrap/db_queries.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Database query utilities for synthetic bootstrap.
+
+All read-only queries against memory.db used by SyntheticBootstrapper.
+These functions are stateless and take db_path as parameter.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+from typing import List, Optional, Set
+
+from .text_utils import clean_fts_query
+
+logger = logging.getLogger("superlocalmemory.learning.bootstrap.db_queries")
+
+
+def get_memory_count(db_path: Path) -> int:
+    """
+    Count total memories in memory.db.
+
+    Args:
+        db_path: Path to memory.db.
+
+    Returns:
+        Total number of memories, or 0 if error.
+    """
+    if not db_path.exists():
+        return 0
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        cursor = conn.cursor()
+        cursor.execute('SELECT COUNT(*) FROM memories')
+        count = cursor.fetchone()[0]
+        conn.close()
+        return count
+    except Exception as e:
+        logger.warning("Failed to count memories: %s", e)
+        return 0
+
+
+def get_memories_by_access(db_path: Path, min_access: int = 5) -> List[dict]:
+    """
+    Fetch memories with access_count >= min_access from memory.db.
+
+    These are memories the user keeps coming back to — strong positive signal.
+
+    Args:
+        db_path: Path to memory.db.
+        min_access: Minimum access_count threshold.
+
+    Returns:
+        List of memory dicts.
+    """
+    if not db_path.exists():
+        return []
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT id, content, summary, project_name, tags,
+                   category, importance, created_at, access_count
+            FROM memories
+            WHERE access_count >= ?
+            ORDER BY access_count DESC
+            LIMIT 100
+        ''', (min_access,))
+        results = [dict(row) for row in cursor.fetchall()]
+        conn.close()
+        return results
+    except Exception as e:
+        logger.warning("Failed to fetch high-access memories: %s", e)
+        return []
+
+
+def get_memories_by_importance(db_path: Path, min_importance: int = 8) -> List[dict]:
+    """
+    Fetch memories with importance >= min_importance from memory.db.
+
+    High importance = user explicitly rated these as valuable.
+
+    Args:
+        db_path: Path to memory.db.
+        min_importance: Minimum importance threshold.
+
+    Returns:
+        List of memory dicts.
+    """
+    if not db_path.exists():
+        return []
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT id, content, summary, project_name, tags,
+                   category, importance, created_at, access_count
+            FROM memories
+            WHERE importance >= ?
+            ORDER BY importance DESC
+            LIMIT 100
+        ''', (min_importance,))
+        results = [dict(row) for row in cursor.fetchall()]
+        conn.close()
+        return results
+    except Exception as e:
+        logger.warning("Failed to fetch high-importance memories: %s", e)
+        return []
+
+
+def get_recent_memories(db_path: Path, limit: int = 30) -> List[dict]:
+    """
+    Fetch the N most recently created memories.
+
+    Args:
+        db_path: Path to memory.db.
+        limit: Maximum number of memories to return.
+
+    Returns:
+        List of memory dicts, sorted by created_at DESC.
+    """
+    if not db_path.exists():
+        return []
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT id, content, summary, project_name, tags,
+                   category, importance, created_at, access_count
+            FROM memories
+            ORDER BY created_at DESC
+            LIMIT ?
+        ''', (limit,))
+        results = [dict(row) for row in cursor.fetchall()]
+        conn.close()
+        return results
+    except Exception as e:
+        logger.warning("Failed to fetch recent memories: %s", e)
+        return []
+
+
+def get_learned_patterns(
+    db_path: Path,
+    min_confidence: float = 0.7,
+) -> List[dict]:
+    """
+    Fetch high-confidence identity_patterns from memory.db.
+
+    These are patterns detected by pattern_learner.py (Layer 4) —
+    tech preferences, coding style, terminology, etc.
+
+    Returns empty list if identity_patterns table doesn't exist
+    (backward compatible with pre-v2.3 databases).
+
+    Args:
+        db_path: Path to memory.db.
+        min_confidence: Minimum confidence threshold.
+
+    Returns:
+        List of pattern dicts.
+    """
+    if not db_path.exists():
+        return []
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        try:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Check if table exists (backward compatibility)
+            cursor.execute('''
+                SELECT name FROM sqlite_master
+                WHERE type='table' AND name='identity_patterns'
+            ''')
+            if cursor.fetchone() is None:
+                return []
+
+            cursor.execute('''
+                SELECT id, pattern_type, key, value, confidence,
+                       evidence_count, category
+                FROM identity_patterns
+                WHERE confidence >= ?
+                ORDER BY confidence DESC
+                LIMIT 50
+            ''', (min_confidence,))
+            results = [dict(row) for row in cursor.fetchall()]
+            return results
+        finally:
+            conn.close()
+    except Exception as e:
+        logger.warning("Failed to fetch learned patterns: %s", e)
+        return []
+
+
+def search_memories(db_path: Path, query: str, limit: int = 20) -> List[dict]:
+    """
+    Simple FTS5 search in memory.db.
+
+    Used to find memories matching synthetic query terms.
+    This is a lightweight search — no TF-IDF, no HNSW, just FTS5.
+
+    Args:
+        db_path: Path to memory.db.
+        query: Search query string.
+        limit: Maximum results to return.
+
+    Returns:
+        List of memory dicts matching the query.
+    """
+    if not db_path.exists():
+        return []
+    if not query or not query.strip():
+        return []
+
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        try:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Clean query for FTS5 (same approach as memory_store_v2.search)
+            fts_query = clean_fts_query(query)
+            if not fts_query:
+                return []
+
+            cursor.execute('''
+                SELECT m.id, m.content, m.summary, m.project_name, m.tags,
+                       m.category, m.importance, m.created_at, m.access_count
+                FROM memories m
+                JOIN memories_fts fts ON m.id = fts.rowid
+                WHERE memories_fts MATCH ?
+                ORDER BY rank
+                LIMIT ?
+            ''', (fts_query, limit))
+            results = [dict(row) for row in cursor.fetchall()]
+            return results
+        finally:
+            conn.close()
+    except Exception as e:
+        logger.debug("FTS5 search failed (may not exist yet): %s", e)
+        return []
+
+
+def find_negative_memories(
+    db_path: Path,
+    anchor_memory: dict,
+    exclude_ids: Optional[Set[int]] = None,
+    limit: int = 2,
+) -> List[dict]:
+    """
+    Find memories dissimilar to the anchor (for negative examples).
+
+    Simple heuristic: pick memories from a different category or project.
+    Falls back to random sample if no structured differences available.
+
+    Args:
+        db_path: Path to memory.db.
+        anchor_memory: The reference memory to find negatives for.
+        exclude_ids: Set of memory IDs to exclude from results.
+        limit: Maximum number of negatives to return.
+
+    Returns:
+        List of negative example memory dicts.
+    """
+    if not db_path.exists():
+        return []
+    exclude_ids = exclude_ids or set()
+
+    try:
+        conn = sqlite3.connect(str(db_path), timeout=5)
+        try:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            anchor_project = anchor_memory.get('project_name', '')
+            anchor_category = anchor_memory.get('category', '')
+
+            # Try to find memories from different project or category
+            conditions = []
+            params: list = []
+
+            if anchor_project:
+                conditions.append('project_name != ?')
+                params.append(anchor_project)
+            if anchor_category:
+                conditions.append('category != ?')
+                params.append(anchor_category)
+
+            # Exclude specified IDs
+            if exclude_ids:
+                placeholders = ','.join('?' for _ in exclude_ids)
+                conditions.append(f'id NOT IN ({placeholders})')
+                params.extend(exclude_ids)
+
+            where_clause = ' AND '.join(conditions) if conditions else '1=1'
+
+            cursor.execute(f'''
+                SELECT id, content, summary, project_name, tags,
+                       category, importance, created_at, access_count
+                FROM memories
+                WHERE {where_clause}
+                ORDER BY RANDOM()
+                LIMIT ?
+            ''', (*params, limit))
+            results = [dict(row) for row in cursor.fetchall()]
+            return results
+        finally:
+            conn.close()
+    except Exception as e:
+        logger.debug("Failed to find negative memories: %s", e)
+        return []

package/src/learning/bootstrap/sampling.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Sampling utilities for synthetic bootstrap.
+
+Functions for diverse sampling and record aggregation.
+"""
+
+from typing import Dict, List
+
+
+def diverse_sample(
+    records: List[dict],
+    target: int,
+) -> List[dict]:
+    """
+    Sample records while maintaining source diversity.
+
+    Takes proportional samples from each source strategy to ensure
+    the training data isn't dominated by one strategy.
+
+    Args:
+        records: List of training records with 'source' field.
+        target: Target number of samples to return.
+
+    Returns:
+        Sampled list of records (at most target items).
+    """
+    if len(records) <= target:
+        return records
+
+    # Group by source
+    by_source: Dict[str, List[dict]] = {}
+    for r in records:
+        src = r.get('source', 'unknown')
+        if src not in by_source:
+            by_source[src] = []
+        by_source[src].append(r)
+
+    # Proportional allocation
+    n_sources = len(by_source)
+    if n_sources == 0:
+        return records[:target]
+
+    per_source = max(1, target // n_sources)
+    sampled = []
+
+    for source, source_records in by_source.items():
+        # Take up to per_source from each, or all if fewer
+        take = min(len(source_records), per_source)
+        sampled.extend(source_records[:take])
+
+    # If under target, fill from remaining
+    if len(sampled) < target:
+        used_ids = {(r['query_hash'], r['memory_id']) for r in sampled}
+        for r in records:
+            if len(sampled) >= target:
+                break
+            key = (r['query_hash'], r['memory_id'])
+            if key not in used_ids:
+                sampled.append(r)
+                used_ids.add(key)
+
+    return sampled[:target]
+
+
+def count_sources(records: List[dict]) -> Dict[str, int]:
+    """
+    Count records by source strategy.
+
+    Args:
+        records: List of training records with 'source' field.
+
+    Returns:
+        Dict mapping source name to count.
+    """
+    counts: Dict[str, int] = {}
+    for r in records:
+        src = r.get('source', 'unknown')
+        counts[src] = counts.get(src, 0) + 1
+    return counts

package/src/learning/bootstrap/text_utils.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Text processing utilities for synthetic bootstrap.
+
+Simple keyword extraction and text processing functions
+with no external NLP dependencies.
+"""
+
+import re
+from collections import Counter
+from typing import List
+
+from .constants import STOPWORDS, MIN_KEYWORD_LENGTH
+
+
+def extract_keywords(content: str, top_n: int = 3) -> List[str]:
+    """
+    Extract meaningful keywords from memory content.
+
+    Simple frequency-based extraction:
+    1. Tokenize (alphanumeric words)
+    2. Remove stopwords and short words
+    3. Return top N by frequency
+
+    No external NLP dependencies — just regex + counter.
+
+    Args:
+        content: Text content to extract keywords from.
+        top_n: Number of top keywords to return.
+
+    Returns:
+        List of top N keywords by frequency.
+    """
+    if not content:
+        return []
+
+    # Tokenize: extract alphanumeric words
+    words = re.findall(r'[a-zA-Z][a-zA-Z0-9_.-]*[a-zA-Z0-9]|[a-zA-Z]', content.lower())
+
+    # Filter stopwords and short words
+    meaningful = [
+        w for w in words
+        if w not in STOPWORDS and len(w) >= MIN_KEYWORD_LENGTH
+    ]
+
+    if not meaningful:
+        return []
+
+    # Count and return top N
+    counter = Counter(meaningful)
+    return [word for word, _count in counter.most_common(top_n)]
+
+
+def clean_fts_query(query: str) -> str:
+    """
+    Clean and prepare query for FTS5 search.
+
+    Extracts word tokens and joins them with OR for FTS5 MATCH syntax.
+
+    Args:
+        query: Raw query string.
+
+    Returns:
+        FTS5-compatible query string, or empty string if no valid tokens.
+    """
+    fts_tokens = re.findall(r'\w+', query)
+    if not fts_tokens:
+        return ''
+    return ' OR '.join(fts_tokens)

package/src/learning/cross_project_aggregator.py

@@ -260,56 +260,55 @@ class CrossProjectAggregator:
 
         try:
             conn = sqlite3.connect(str(self.memory_db_path), timeout=10)
-            conn.execute("PRAGMA busy_timeout=5000")
-            cursor = conn.cursor()
-
-            # Get distinct profiles
-            cursor.execute(
-                "SELECT DISTINCT profile FROM memories "
-                "WHERE profile IS NOT NULL ORDER BY profile"
-            )
-            profiles = [row[0] for row in cursor.fetchall()]
-
-            if not profiles:
-                # Fallback: if no profile column or all NULL, treat as 'default'
-                cursor.execute("SELECT id FROM memories ORDER BY created_at")
-                all_ids = [row[0] for row in cursor.fetchall()]
-                if all_ids:
-                    # Get the latest timestamp
-                    cursor.execute(
-                        "SELECT MAX(created_at) FROM memories"
-                    )
-                    latest = cursor.fetchone()[0] or datetime.now().isoformat()
-                    results.append({
-                        "profile": "default",
-                        "memory_ids": all_ids,
-                        "latest_timestamp": latest,
-                    })
-                conn.close()
-                return results
-
-            # For each profile, get memory IDs and latest timestamp
-            for profile in profiles:
-                cursor.execute(
-                    "SELECT id FROM memories WHERE profile = ? ORDER BY created_at",
-                    (profile,),
-                )
-                memory_ids = [row[0] for row in cursor.fetchall()]
+            try:
+                conn.execute("PRAGMA busy_timeout=5000")
+                cursor = conn.cursor()
 
+                # Get distinct profiles
                 cursor.execute(
-                    "SELECT MAX(created_at) FROM memories WHERE profile = ?",
-                    (profile,),
+                    "SELECT DISTINCT profile FROM memories "
+                    "WHERE profile IS NOT NULL ORDER BY profile"
                 )
-                latest = cursor.fetchone()[0] or datetime.now().isoformat()
+                profiles = [row[0] for row in cursor.fetchall()]
+
+                if not profiles:
+                    # Fallback: if no profile column or all NULL, treat as 'default'
+                    cursor.execute("SELECT id FROM memories ORDER BY created_at")
+                    all_ids = [row[0] for row in cursor.fetchall()]
+                    if all_ids:
+                        # Get the latest timestamp
+                        cursor.execute(
+                            "SELECT MAX(created_at) FROM memories"
+                        )
+                        latest = cursor.fetchone()[0] or datetime.now().isoformat()
+                        results.append({
+                            "profile": "default",
+                            "memory_ids": all_ids,
+                            "latest_timestamp": latest,
+                        })
+
+                # For each profile, get memory IDs and latest timestamp
+                for profile in profiles:
+                    cursor.execute(
+                        "SELECT id FROM memories WHERE profile = ? ORDER BY created_at",
+                        (profile,),
+                    )
+                    memory_ids = [row[0] for row in cursor.fetchall()]
 
-                if memory_ids:
-                    results.append({
-                        "profile": profile,
-                        "memory_ids": memory_ids,
-                        "latest_timestamp": latest,
-                    })
+                    cursor.execute(
+                        "SELECT MAX(created_at) FROM memories WHERE profile = ?",
+                        (profile,),
+                    )
+                    latest = cursor.fetchone()[0] or datetime.now().isoformat()
 
-            conn.close()
+                    if memory_ids:
+                        results.append({
+                            "profile": profile,
+                            "memory_ids": memory_ids,
+                            "latest_timestamp": latest,
+                        })
+            finally:
+                conn.close()
 
         except sqlite3.OperationalError as e:
             # Handle case where 'profile' column doesn't exist
@@ -320,18 +319,20 @@ class CrossProjectAggregator:
             )
             try:
                 conn = sqlite3.connect(str(self.memory_db_path), timeout=10)
-                cursor = conn.cursor()
-                cursor.execute("SELECT id FROM memories ORDER BY created_at")
-                all_ids = [row[0] for row in cursor.fetchall()]
-                if all_ids:
-                    cursor.execute("SELECT MAX(created_at) FROM memories")
-                    latest = cursor.fetchone()[0] or datetime.now().isoformat()
-                    results.append({
-                        "profile": "default",
-                        "memory_ids": all_ids,
-                        "latest_timestamp": latest,
-                    })
-                conn.close()
+                try:
+                    cursor = conn.cursor()
+                    cursor.execute("SELECT id FROM memories ORDER BY created_at")
+                    all_ids = [row[0] for row in cursor.fetchall()]
+                    if all_ids:
+                        cursor.execute("SELECT MAX(created_at) FROM memories")
+                        latest = cursor.fetchone()[0] or datetime.now().isoformat()
+                        results.append({
+                            "profile": "default",
+                            "memory_ids": all_ids,
+                            "latest_timestamp": latest,
+                        })
+                finally:
+                    conn.close()
             except Exception as inner_e:
                 logger.error("Failed to read memory.db: %s", inner_e)
 

package/src/learning/db/__init__.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Learning database utilities and schema management.
+
+This package contains extracted modules from learning_db.py to improve
+code organization and maintainability.
+"""
+
+from .constants import (
+    MEMORY_DIR,
+    LEARNING_DB_PATH,
+    DEFAULT_PROFILE,
+    DEFAULT_CONFIDENCE,
+    DEFAULT_LIMIT,
+    ALL_TABLES,
+)
+
+from .schema import (
+    initialize_schema,
+    create_all_tables,
+    add_profile_columns,
+    create_indexes,
+)
+
+__all__ = [
+    # Constants
+    "MEMORY_DIR",
+    "LEARNING_DB_PATH",
+    "DEFAULT_PROFILE",
+    "DEFAULT_CONFIDENCE",
+    "DEFAULT_LIMIT",
+    "ALL_TABLES",
+    # Schema functions
+    "initialize_schema",
+    "create_all_tables",
+    "add_profile_columns",
+    "create_indexes",
+]

package/src/learning/db/constants.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Constants and defaults for learning.db.
+
+This module contains database paths, configuration defaults, and other
+constants used by the learning database system.
+"""
+
+from pathlib import Path
+
+# Database paths
+MEMORY_DIR = Path.home() / ".claude-memory"
+LEARNING_DB_PATH = MEMORY_DIR / "learning.db"
+
+# Default values
+DEFAULT_PROFILE = "default"
+DEFAULT_CONFIDENCE = 1.0
+DEFAULT_LIMIT = 100
+
+# Table names (for reference and testing)
+TABLE_TRANSFERABLE_PATTERNS = "transferable_patterns"
+TABLE_WORKFLOW_PATTERNS = "workflow_patterns"
+TABLE_RANKING_FEEDBACK = "ranking_feedback"
+TABLE_RANKING_MODELS = "ranking_models"
+TABLE_SOURCE_QUALITY = "source_quality"
+TABLE_ENGAGEMENT_METRICS = "engagement_metrics"
+TABLE_ACTION_OUTCOMES = "action_outcomes"
+TABLE_BEHAVIORAL_PATTERNS = "behavioral_patterns"
+TABLE_CROSS_PROJECT_BEHAVIORS = "cross_project_behaviors"
+
+# All table names for iteration
+ALL_TABLES = [
+    TABLE_TRANSFERABLE_PATTERNS,
+    TABLE_WORKFLOW_PATTERNS,
+    TABLE_RANKING_FEEDBACK,
+    TABLE_RANKING_MODELS,
+    TABLE_SOURCE_QUALITY,
+    TABLE_ENGAGEMENT_METRICS,
+    TABLE_ACTION_OUTCOMES,
+    TABLE_BEHAVIORAL_PATTERNS,
+    TABLE_CROSS_PROJECT_BEHAVIORS,
+]