superlocalmemory 2.8.5 → 3.0.0

package/src/learning/bootstrap/__init__.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Bootstrap utilities package.
-
-Re-exports all constants, functions, and utilities used by SyntheticBootstrapper.
-"""
-
-# Constants
-from .constants import (
-    MEMORY_DB_PATH,
-    MODELS_DIR,
-    MODEL_PATH,
-    MIN_MEMORIES_FOR_BOOTSTRAP,
-    BOOTSTRAP_CONFIG,
-    BOOTSTRAP_PARAMS,
-    STOPWORDS,
-    MIN_KEYWORD_LENGTH,
-)
-
-# Text utilities
-from .text_utils import (
-    extract_keywords,
-    clean_fts_query,
-)
-
-# Database queries
-from .db_queries import (
-    get_memory_count,
-    get_memories_by_access,
-    get_memories_by_importance,
-    get_recent_memories,
-    get_learned_patterns,
-    search_memories,
-    find_negative_memories,
-)
-
-# Sampling utilities
-from .sampling import (
-    diverse_sample,
-    count_sources,
-)
-
-__all__ = [
-    # Constants
-    'MEMORY_DB_PATH',
-    'MODELS_DIR',
-    'MODEL_PATH',
-    'MIN_MEMORIES_FOR_BOOTSTRAP',
-    'BOOTSTRAP_CONFIG',
-    'BOOTSTRAP_PARAMS',
-    'STOPWORDS',
-    'MIN_KEYWORD_LENGTH',
-    # Text utilities
-    'extract_keywords',
-    'clean_fts_query',
-    # Database queries
-    'get_memory_count',
-    'get_memories_by_access',
-    'get_memories_by_importance',
-    'get_recent_memories',
-    'get_learned_patterns',
-    'search_memories',
-    'find_negative_memories',
-    # Sampling
-    'diverse_sample',
-    'count_sources',
-]

package/src/learning/bootstrap/constants.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Bootstrap constants and configuration.
-
-All constant values, configuration dicts, and static data used
-by SyntheticBootstrapper are defined here.
-"""
-
-from pathlib import Path
-
-# ============================================================================
-# Paths
-# ============================================================================
-
-MEMORY_DB_PATH = Path.home() / ".claude-memory" / "memory.db"
-MODELS_DIR = Path.home() / ".claude-memory" / "models"
-MODEL_PATH = MODELS_DIR / "ranker.txt"
-
-# ============================================================================
-# Bootstrap Configuration
-# ============================================================================
-
-# Minimum memories needed before bootstrap makes sense
-MIN_MEMORIES_FOR_BOOTSTRAP = 50
-
-# Tiered config — bootstrap model complexity scales with data size
-BOOTSTRAP_CONFIG = {
-    'small': {
-        'min_memories': 50,
-        'max_memories': 499,
-        'target_samples': 200,
-        'n_estimators': 30,
-        'max_depth': 3,
-    },
-    'medium': {
-        'min_memories': 500,
-        'max_memories': 4999,
-        'target_samples': 1000,
-        'n_estimators': 50,
-        'max_depth': 4,
-    },
-    'large': {
-        'min_memories': 5000,
-        'max_memories': float('inf'),
-        'target_samples': 2000,
-        'n_estimators': 100,
-        'max_depth': 6,
-    },
-}
-
-# ============================================================================
-# LightGBM Parameters
-# ============================================================================
-
-# LightGBM bootstrap parameters — MORE aggressive regularization than
-# real training because synthetic data has systematic biases
-BOOTSTRAP_PARAMS = {
-    'objective': 'lambdarank',
-    'metric': 'ndcg',
-    'ndcg_eval_at': [5, 10],
-    'learning_rate': 0.1,
-    'num_leaves': 8,
-    'max_depth': 3,
-    'min_child_samples': 5,
-    'subsample': 0.7,
-    'reg_alpha': 0.5,
-    'reg_lambda': 2.0,
-    'boosting_type': 'dart',
-    'verbose': -1,
-}
-
-# ============================================================================
-# Text Processing
-# ============================================================================
-
-# English stopwords for keyword extraction (no external deps)
-STOPWORDS = frozenset({
-    'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
-    'of', 'with', 'by', 'from', 'is', 'it', 'this', 'that', 'was', 'are',
-    'be', 'has', 'have', 'had', 'do', 'does', 'did', 'will', 'would',
-    'could', 'should', 'may', 'might', 'can', 'not', 'no', 'if', 'then',
-    'so', 'as', 'up', 'out', 'about', 'into', 'over', 'after', 'before',
-    'when', 'where', 'how', 'what', 'which', 'who', 'whom', 'why',
-    'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other',
-    'some', 'such', 'than', 'too', 'very', 'just', 'also', 'now',
-    'here', 'there', 'use', 'used', 'using', 'make', 'made',
-    'need', 'needed', 'get', 'got', 'set', 'new', 'old', 'one', 'two',
-})
-
-# Minimum word length for keyword extraction
-MIN_KEYWORD_LENGTH = 3

package/src/learning/bootstrap/db_queries.py

@@ -1,316 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Database query utilities for synthetic bootstrap.
-
-All read-only queries against memory.db used by SyntheticBootstrapper.
-These functions are stateless and take db_path as parameter.
-"""
-
-import logging
-import sqlite3
-from pathlib import Path
-from typing import List, Optional, Set
-
-from .text_utils import clean_fts_query
-
-logger = logging.getLogger("superlocalmemory.learning.bootstrap.db_queries")
-
-
-def get_memory_count(db_path: Path) -> int:
-    """
-    Count total memories in memory.db.
-
-    Args:
-        db_path: Path to memory.db.
-
-    Returns:
-        Total number of memories, or 0 if error.
-    """
-    if not db_path.exists():
-        return 0
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        cursor = conn.cursor()
-        cursor.execute('SELECT COUNT(*) FROM memories')
-        count = cursor.fetchone()[0]
-        conn.close()
-        return count
-    except Exception as e:
-        logger.warning("Failed to count memories: %s", e)
-        return 0
-
-
-def get_memories_by_access(db_path: Path, min_access: int = 5) -> List[dict]:
-    """
-    Fetch memories with access_count >= min_access from memory.db.
-
-    These are memories the user keeps coming back to — strong positive signal.
-
-    Args:
-        db_path: Path to memory.db.
-        min_access: Minimum access_count threshold.
-
-    Returns:
-        List of memory dicts.
-    """
-    if not db_path.exists():
-        return []
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        conn.row_factory = sqlite3.Row
-        cursor = conn.cursor()
-        cursor.execute('''
-            SELECT id, content, summary, project_name, tags,
-                   category, importance, created_at, access_count
-            FROM memories
-            WHERE access_count >= ?
-            ORDER BY access_count DESC
-            LIMIT 100
-        ''', (min_access,))
-        results = [dict(row) for row in cursor.fetchall()]
-        conn.close()
-        return results
-    except Exception as e:
-        logger.warning("Failed to fetch high-access memories: %s", e)
-        return []
-
-
-def get_memories_by_importance(db_path: Path, min_importance: int = 8) -> List[dict]:
-    """
-    Fetch memories with importance >= min_importance from memory.db.
-
-    High importance = user explicitly rated these as valuable.
-
-    Args:
-        db_path: Path to memory.db.
-        min_importance: Minimum importance threshold.
-
-    Returns:
-        List of memory dicts.
-    """
-    if not db_path.exists():
-        return []
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        conn.row_factory = sqlite3.Row
-        cursor = conn.cursor()
-        cursor.execute('''
-            SELECT id, content, summary, project_name, tags,
-                   category, importance, created_at, access_count
-            FROM memories
-            WHERE importance >= ?
-            ORDER BY importance DESC
-            LIMIT 100
-        ''', (min_importance,))
-        results = [dict(row) for row in cursor.fetchall()]
-        conn.close()
-        return results
-    except Exception as e:
-        logger.warning("Failed to fetch high-importance memories: %s", e)
-        return []
-
-
-def get_recent_memories(db_path: Path, limit: int = 30) -> List[dict]:
-    """
-    Fetch the N most recently created memories.
-
-    Args:
-        db_path: Path to memory.db.
-        limit: Maximum number of memories to return.
-
-    Returns:
-        List of memory dicts, sorted by created_at DESC.
-    """
-    if not db_path.exists():
-        return []
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        conn.row_factory = sqlite3.Row
-        cursor = conn.cursor()
-        cursor.execute('''
-            SELECT id, content, summary, project_name, tags,
-                   category, importance, created_at, access_count
-            FROM memories
-            ORDER BY created_at DESC
-            LIMIT ?
-        ''', (limit,))
-        results = [dict(row) for row in cursor.fetchall()]
-        conn.close()
-        return results
-    except Exception as e:
-        logger.warning("Failed to fetch recent memories: %s", e)
-        return []
-
-
-def get_learned_patterns(
-    db_path: Path,
-    min_confidence: float = 0.7,
-) -> List[dict]:
-    """
-    Fetch high-confidence identity_patterns from memory.db.
-
-    These are patterns detected by pattern_learner.py (Layer 4) —
-    tech preferences, coding style, terminology, etc.
-
-    Returns empty list if identity_patterns table doesn't exist
-    (backward compatible with pre-v2.3 databases).
-
-    Args:
-        db_path: Path to memory.db.
-        min_confidence: Minimum confidence threshold.
-
-    Returns:
-        List of pattern dicts.
-    """
-    if not db_path.exists():
-        return []
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        try:
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            # Check if table exists (backward compatibility)
-            cursor.execute('''
-                SELECT name FROM sqlite_master
-                WHERE type='table' AND name='identity_patterns'
-            ''')
-            if cursor.fetchone() is None:
-                return []
-
-            cursor.execute('''
-                SELECT id, pattern_type, key, value, confidence,
-                       evidence_count, category
-                FROM identity_patterns
-                WHERE confidence >= ?
-                ORDER BY confidence DESC
-                LIMIT 50
-            ''', (min_confidence,))
-            results = [dict(row) for row in cursor.fetchall()]
-            return results
-        finally:
-            conn.close()
-    except Exception as e:
-        logger.warning("Failed to fetch learned patterns: %s", e)
-        return []
-
-
-def search_memories(db_path: Path, query: str, limit: int = 20) -> List[dict]:
-    """
-    Simple FTS5 search in memory.db.
-
-    Used to find memories matching synthetic query terms.
-    This is a lightweight search — no TF-IDF, no HNSW, just FTS5.
-
-    Args:
-        db_path: Path to memory.db.
-        query: Search query string.
-        limit: Maximum results to return.
-
-    Returns:
-        List of memory dicts matching the query.
-    """
-    if not db_path.exists():
-        return []
-    if not query or not query.strip():
-        return []
-
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        try:
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            # Clean query for FTS5 (same approach as memory_store_v2.search)
-            fts_query = clean_fts_query(query)
-            if not fts_query:
-                return []
-
-            cursor.execute('''
-                SELECT m.id, m.content, m.summary, m.project_name, m.tags,
-                       m.category, m.importance, m.created_at, m.access_count
-                FROM memories m
-                JOIN memories_fts fts ON m.id = fts.rowid
-                WHERE memories_fts MATCH ?
-                ORDER BY rank
-                LIMIT ?
-            ''', (fts_query, limit))
-            results = [dict(row) for row in cursor.fetchall()]
-            return results
-        finally:
-            conn.close()
-    except Exception as e:
-        logger.debug("FTS5 search failed (may not exist yet): %s", e)
-        return []
-
-
-def find_negative_memories(
-    db_path: Path,
-    anchor_memory: dict,
-    exclude_ids: Optional[Set[int]] = None,
-    limit: int = 2,
-) -> List[dict]:
-    """
-    Find memories dissimilar to the anchor (for negative examples).
-
-    Simple heuristic: pick memories from a different category or project.
-    Falls back to random sample if no structured differences available.
-
-    Args:
-        db_path: Path to memory.db.
-        anchor_memory: The reference memory to find negatives for.
-        exclude_ids: Set of memory IDs to exclude from results.
-        limit: Maximum number of negatives to return.
-
-    Returns:
-        List of negative example memory dicts.
-    """
-    if not db_path.exists():
-        return []
-    exclude_ids = exclude_ids or set()
-
-    try:
-        conn = sqlite3.connect(str(db_path), timeout=5)
-        try:
-            conn.row_factory = sqlite3.Row
-            cursor = conn.cursor()
-
-            anchor_project = anchor_memory.get('project_name', '')
-            anchor_category = anchor_memory.get('category', '')
-
-            # Try to find memories from different project or category
-            conditions = []
-            params: list = []
-
-            if anchor_project:
-                conditions.append('project_name != ?')
-                params.append(anchor_project)
-            if anchor_category:
-                conditions.append('category != ?')
-                params.append(anchor_category)
-
-            # Exclude specified IDs
-            if exclude_ids:
-                placeholders = ','.join('?' for _ in exclude_ids)
-                conditions.append(f'id NOT IN ({placeholders})')
-                params.extend(exclude_ids)
-
-            where_clause = ' AND '.join(conditions) if conditions else '1=1'
-
-            cursor.execute(f'''
-                SELECT id, content, summary, project_name, tags,
-                       category, importance, created_at, access_count
-                FROM memories
-                WHERE {where_clause}
-                ORDER BY RANDOM()
-                LIMIT ?
-            ''', (*params, limit))
-            results = [dict(row) for row in cursor.fetchall()]
-            return results
-        finally:
-            conn.close()
-    except Exception as e:
-        logger.debug("Failed to find negative memories: %s", e)
-        return []

package/src/learning/bootstrap/sampling.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Sampling utilities for synthetic bootstrap.
-
-Functions for diverse sampling and record aggregation.
-"""
-
-from typing import Dict, List
-
-
-def diverse_sample(
-    records: List[dict],
-    target: int,
-) -> List[dict]:
-    """
-    Sample records while maintaining source diversity.
-
-    Takes proportional samples from each source strategy to ensure
-    the training data isn't dominated by one strategy.
-
-    Args:
-        records: List of training records with 'source' field.
-        target: Target number of samples to return.
-
-    Returns:
-        Sampled list of records (at most target items).
-    """
-    if len(records) <= target:
-        return records
-
-    # Group by source
-    by_source: Dict[str, List[dict]] = {}
-    for r in records:
-        src = r.get('source', 'unknown')
-        if src not in by_source:
-            by_source[src] = []
-        by_source[src].append(r)
-
-    # Proportional allocation
-    n_sources = len(by_source)
-    if n_sources == 0:
-        return records[:target]
-
-    per_source = max(1, target // n_sources)
-    sampled = []
-
-    for source, source_records in by_source.items():
-        # Take up to per_source from each, or all if fewer
-        take = min(len(source_records), per_source)
-        sampled.extend(source_records[:take])
-
-    # If under target, fill from remaining
-    if len(sampled) < target:
-        used_ids = {(r['query_hash'], r['memory_id']) for r in sampled}
-        for r in records:
-            if len(sampled) >= target:
-                break
-            key = (r['query_hash'], r['memory_id'])
-            if key not in used_ids:
-                sampled.append(r)
-                used_ids.add(key)
-
-    return sampled[:target]
-
-
-def count_sources(records: List[dict]) -> Dict[str, int]:
-    """
-    Count records by source strategy.
-
-    Args:
-        records: List of training records with 'source' field.
-
-    Returns:
-        Dict mapping source name to count.
-    """
-    counts: Dict[str, int] = {}
-    for r in records:
-        src = r.get('source', 'unknown')
-        counts[src] = counts.get(src, 0) + 1
-    return counts

package/src/learning/bootstrap/text_utils.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Text processing utilities for synthetic bootstrap.
-
-Simple keyword extraction and text processing functions
-with no external NLP dependencies.
-"""
-
-import re
-from collections import Counter
-from typing import List
-
-from .constants import STOPWORDS, MIN_KEYWORD_LENGTH
-
-
-def extract_keywords(content: str, top_n: int = 3) -> List[str]:
-    """
-    Extract meaningful keywords from memory content.
-
-    Simple frequency-based extraction:
-    1. Tokenize (alphanumeric words)
-    2. Remove stopwords and short words
-    3. Return top N by frequency
-
-    No external NLP dependencies — just regex + counter.
-
-    Args:
-        content: Text content to extract keywords from.
-        top_n: Number of top keywords to return.
-
-    Returns:
-        List of top N keywords by frequency.
-    """
-    if not content:
-        return []
-
-    # Tokenize: extract alphanumeric words
-    words = re.findall(r'[a-zA-Z][a-zA-Z0-9_.-]*[a-zA-Z0-9]|[a-zA-Z]', content.lower())
-
-    # Filter stopwords and short words
-    meaningful = [
-        w for w in words
-        if w not in STOPWORDS and len(w) >= MIN_KEYWORD_LENGTH
-    ]
-
-    if not meaningful:
-        return []
-
-    # Count and return top N
-    counter = Counter(meaningful)
-    return [word for word, _count in counter.most_common(top_n)]
-
-
-def clean_fts_query(query: str) -> str:
-    """
-    Clean and prepare query for FTS5 search.
-
-    Extracts word tokens and joins them with OR for FTS5 MATCH syntax.
-
-    Args:
-        query: Raw query string.
-
-    Returns:
-        FTS5-compatible query string, or empty string if no valid tokens.
-    """
-    fts_tokens = re.findall(r'\w+', query)
-    if not fts_tokens:
-        return ''
-    return ' OR '.join(fts_tokens)