superlocalmemory 2.8.6 → 3.0.0

package/src/patterns/store.py

@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Pattern Store - SQLite-backed pattern storage and retrieval.
-
-Handles identity_patterns and pattern_examples tables,
-including schema migration, CRUD operations, and profile-scoped queries.
-"""
-
-import sqlite3
-import json
-import logging
-from typing import Dict, List, Optional, Any
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-
-class PatternStore:
-    """Handles pattern storage and retrieval."""
-
-    def __init__(self, db_path: Path):
-        self.db_path = db_path
-        self._init_tables()
-
-    def _init_tables(self):
-        """Initialize pattern tables if they don't exist, or recreate if schema is incomplete."""
-        conn = sqlite3.connect(self.db_path)
-        cursor = conn.cursor()
-
-        # Check if existing tables have correct schema
-        for table_name, required_cols in [
-            ('identity_patterns', {'pattern_type', 'key', 'value', 'confidence'}),
-            ('pattern_examples', {'pattern_id', 'memory_id'}),
-        ]:
-            cursor.execute(f"PRAGMA table_info({table_name})")
-            existing_cols = {row[1] for row in cursor.fetchall()}
-            if existing_cols and not required_cols.issubset(existing_cols):
-                logger.warning(f"Dropping incomplete {table_name} table (missing: {required_cols - existing_cols})")
-                cursor.execute(f'DROP TABLE IF EXISTS {table_name}')
-
-        # Identity patterns table
-        cursor.execute('''
-            CREATE TABLE IF NOT EXISTS identity_patterns (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                pattern_type TEXT NOT NULL,
-                key TEXT NOT NULL,
-                value TEXT NOT NULL,
-                confidence REAL DEFAULT 0.5,
-                evidence_count INTEGER DEFAULT 1,
-                memory_ids TEXT,
-                category TEXT,
-                profile TEXT DEFAULT 'default',
-                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                UNIQUE(pattern_type, key, category, profile)
-            )
-        ''')
-
-        # Add profile column if upgrading from older schema
-        try:
-            cursor.execute('ALTER TABLE identity_patterns ADD COLUMN profile TEXT DEFAULT "default"')
-        except sqlite3.OperationalError:
-            pass  # Column already exists
-
-        # Pattern examples table
-        cursor.execute('''
-            CREATE TABLE IF NOT EXISTS pattern_examples (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                pattern_id INTEGER NOT NULL,
-                memory_id INTEGER NOT NULL,
-                example_text TEXT,
-                FOREIGN KEY (pattern_id) REFERENCES identity_patterns(id) ON DELETE CASCADE,
-                FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
-            )
-        ''')
-
-        # Indexes
-        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_type ON identity_patterns(pattern_type)')
-        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_confidence ON identity_patterns(confidence)')
-        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_profile ON identity_patterns(profile)')
-
-        conn.commit()
-        conn.close()
-
-    def save_pattern(self, pattern: Dict[str, Any]) -> int:
-        """Save or update a pattern (scoped by profile)."""
-        conn = sqlite3.connect(self.db_path)
-        cursor = conn.cursor()
-        profile = pattern.get('profile', 'default')
-
-        try:
-            # Check if pattern exists for this profile
-            cursor.execute('''
-                SELECT id FROM identity_patterns
-                WHERE pattern_type = ? AND key = ? AND category = ? AND profile = ?
-            ''', (pattern['pattern_type'], pattern['key'], pattern['category'], profile))
-
-            existing = cursor.fetchone()
-
-            memory_ids_json = json.dumps(pattern['memory_ids'])
-
-            if existing:
-                # Update existing pattern
-                pattern_id = existing[0]
-                cursor.execute('''
-                    UPDATE identity_patterns
-                    SET value = ?, confidence = ?, evidence_count = ?,
-                        memory_ids = ?, updated_at = CURRENT_TIMESTAMP
-                    WHERE id = ?
-                ''', (
-                    pattern['value'],
-                    pattern['confidence'],
-                    pattern['evidence_count'],
-                    memory_ids_json,
-                    pattern_id
-                ))
-            else:
-                # Insert new pattern
-                cursor.execute('''
-                    INSERT INTO identity_patterns
-                    (pattern_type, key, value, confidence, evidence_count, memory_ids, category, profile)
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-                ''', (
-                    pattern['pattern_type'],
-                    pattern['key'],
-                    pattern['value'],
-                    pattern['confidence'],
-                    pattern['evidence_count'],
-                    memory_ids_json,
-                    pattern['category'],
-                    profile
-                ))
-                pattern_id = cursor.lastrowid
-
-            # Save examples
-            self._save_pattern_examples(cursor, pattern_id, pattern['memory_ids'], pattern['key'])
-
-            conn.commit()
-            return pattern_id
-
-        finally:
-            conn.close()
-
-    def _save_pattern_examples(self, cursor, pattern_id: int, memory_ids: List[int], key: str):
-        """Save representative examples for pattern."""
-        # Clear old examples
-        cursor.execute('DELETE FROM pattern_examples WHERE pattern_id = ?', (pattern_id,))
-
-        # Save top 3 examples
-        for memory_id in memory_ids[:3]:
-            cursor.execute('SELECT content FROM memories WHERE id = ?', (memory_id,))
-            row = cursor.fetchone()
-
-            if row:
-                content = row[0]
-                excerpt = self._extract_relevant_excerpt(content, key)
-
-                cursor.execute('''
-                    INSERT INTO pattern_examples (pattern_id, memory_id, example_text)
-                    VALUES (?, ?, ?)
-                ''', (pattern_id, memory_id, excerpt))
-
-    def _extract_relevant_excerpt(self, content: str, key: str) -> str:
-        """Extract 150-char excerpt showing pattern."""
-        # Find first mention of key term
-        key_lower = key.lower().replace('_', ' ')
-        idx = content.lower().find(key_lower)
-
-        if idx >= 0:
-            start = max(0, idx - 50)
-            end = min(len(content), idx + 100)
-            excerpt = content[start:end]
-            return excerpt if len(excerpt) <= 150 else excerpt[:150] + '...'
-
-        # Fallback: first 150 chars
-        return content[:150] + ('...' if len(content) > 150 else '')
-
-    def get_patterns(self, min_confidence: float = 0.7, pattern_type: Optional[str] = None,
-                     profile: Optional[str] = None) -> List[Dict[str, Any]]:
-        """Get patterns above confidence threshold, optionally filtered by profile."""
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            # Build query with optional filters
-            conditions = ['confidence >= ?']
-            params = [min_confidence]
-
-            if pattern_type:
-                conditions.append('pattern_type = ?')
-                params.append(pattern_type)
-
-            if profile:
-                conditions.append('profile = ?')
-                params.append(profile)
-
-            where_clause = ' AND '.join(conditions)
-            cursor.execute(f'''
-                SELECT id, pattern_type, key, value, confidence, evidence_count,
-                       updated_at, created_at, category
-                FROM identity_patterns
-                WHERE {where_clause}
-                ORDER BY confidence DESC, evidence_count DESC
-            ''', params)
-
-            patterns = []
-            for row in cursor.fetchall():
-                patterns.append({
-                    'id': row[0],
-                    'pattern_type': row[1],
-                    'key': row[2],
-                    'value': row[3],
-                    'confidence': row[4],
-                    'evidence_count': row[5],
-                    'frequency': row[5],
-                    'last_seen': row[6],
-                    'created_at': row[7],
-                    'category': row[8]
-                })
-
-        finally:
-            conn.close()
-        return patterns

package/src/patterns/terminology.py

@@ -1,140 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-Terminology Learner - User-specific term definition extraction.
-
-Learns how the user defines ambiguous terms like 'optimize', 'refactor', etc.
-by analyzing contextual co-occurrence patterns across memories.
-"""
-
-import sqlite3
-import re
-import logging
-from typing import Dict, List, Optional, Any
-from collections import Counter
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-
-class TerminologyLearner:
-    """Learns user-specific definitions of common terms."""
-
-    def __init__(self, db_path: Path):
-        self.db_path = db_path
-
-        # Common ambiguous terms to learn
-        self.ambiguous_terms = [
-            'optimize', 'refactor', 'clean', 'simple',
-            'mvp', 'prototype', 'scale', 'production-ready',
-            'fix', 'improve', 'update', 'enhance'
-        ]
-
-    def learn_terminology(self, memory_ids: List[int]) -> Dict[str, Dict[str, Any]]:
-        """Learn user-specific term definitions."""
-        patterns = {}
-
-        conn = sqlite3.connect(self.db_path)
-        try:
-            cursor = conn.cursor()
-
-            for term in self.ambiguous_terms:
-                contexts = []
-
-                # Find all contexts where term appears
-                for memory_id in memory_ids:
-                    cursor.execute('SELECT content FROM memories WHERE id = ?', (memory_id,))
-                    row = cursor.fetchone()
-
-                    if not row:
-                        continue
-
-                    content = row[0]
-
-                    # Find term in content (case-insensitive)
-                    pattern = r'\b' + re.escape(term) + r'\b'
-                    for match in re.finditer(pattern, content, re.IGNORECASE):
-                        term_idx = match.start()
-
-                        # Extract 100-char window around term
-                        start = max(0, term_idx - 100)
-                        end = min(len(content), term_idx + len(term) + 100)
-                        context_window = content[start:end]
-
-                        contexts.append({
-                            'memory_id': memory_id,
-                            'context': context_window
-                        })
-
-                # Analyze contexts to extract meaning (need at least 3 examples)
-                if len(contexts) >= 3:
-                    definition = self._extract_definition(term, contexts)
-
-                    if definition:
-                        evidence_list = list(set([ctx['memory_id'] for ctx in contexts]))
-
-                        # Confidence increases with more examples, capped at 0.95
-                        confidence = min(0.95, 0.6 + (len(contexts) * 0.05))
-
-                        patterns[term] = {
-                            'pattern_type': 'terminology',
-                            'key': term,
-                            'value': definition,
-                            'confidence': round(confidence, 2),
-                            'evidence_count': len(evidence_list),
-                            'memory_ids': evidence_list,
-                            'category': 'general'
-                        }
-
-        finally:
-            conn.close()
-        return patterns
-
-    def _extract_definition(self, term: str, contexts: List[Dict]) -> Optional[str]:
-        """Extract definition from contexts using pattern matching."""
-        # Collect words near the term across all contexts
-        nearby_words = []
-
-        for ctx in contexts:
-            words = re.findall(r'\b\w+\b', ctx['context'].lower())
-            nearby_words.extend(words)
-
-        # Count word frequencies
-        word_counts = Counter(nearby_words)
-
-        # Remove the term itself and common stopwords
-        stopwords = {'the', 'a', 'an', 'is', 'to', 'for', 'of', 'in', 'on', 'at',
-                     'and', 'or', 'but', 'with', 'from', 'by', 'this', 'that'}
-        word_counts = Counter({w: c for w, c in word_counts.items()
-                              if w not in stopwords and w != term.lower()})
-
-        # Get top co-occurring words
-        top_words = [w for w, _ in word_counts.most_common(8)]
-
-        # Apply heuristic rules based on term and context
-        if term == 'optimize':
-            if any(w in top_words for w in ['performance', 'speed', 'faster', 'latency']):
-                return "Performance optimization (speed/latency)"
-            elif any(w in top_words for w in ['code', 'clean', 'refactor']):
-                return "Code quality optimization"
-
-        elif term == 'refactor':
-            if any(w in top_words for w in ['architecture', 'structure', 'design']):
-                return "Architecture change, not just renaming"
-            elif any(w in top_words for w in ['clean', 'organize', 'simplify']):
-                return "Code organization improvement"
-
-        elif term == 'mvp':
-            if any(w in top_words for w in ['core', 'basic', 'essential', 'minimal']):
-                return "Core features only, no polish"
-
-        elif term == 'production-ready':
-            if any(w in top_words for w in ['test', 'error', 'monitoring', 'deploy']):
-                return "Fully tested and monitored for deployment"
-
-        # Generic definition if specific pattern not matched
-        if len(top_words) >= 3:
-            return f"Commonly used with: {', '.join(top_words[:3])}"
-
-        return None

package/src/provenance_tracker.py

@@ -1,312 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
-"""
-ProvenanceTracker — Tracks the origin and lineage of every memory.
-
-Adds provenance columns to the memories table:
-    created_by       — Agent ID that created this memory (e.g., "mcp:claude-desktop")
-    source_protocol  — Protocol used (mcp, cli, rest, python)
-    trust_score      — Trust score at time of creation (default 1.0)
-    provenance_chain — JSON array of derivation history
-
-This enables:
-    - "Who wrote this?" queries for the dashboard
-    - Trust-weighted recall (v2.6 — higher trust = higher ranking)
-    - Audit trail for enterprise compliance (v3.0)
-    - Memory lineage tracking (if agent B derives from agent A's memory)
-
-Column migration is safe: uses ALTER TABLE ADD COLUMN with try/except.
-Old databases without provenance columns work fine — values default.
-"""
-
-import json
-import logging
-import sqlite3
-import threading
-from datetime import datetime
-from pathlib import Path
-from typing import Optional, Dict, Any
-
-logger = logging.getLogger("superlocalmemory.provenance")
-
-
-class ProvenanceTracker:
-    """
-    Tracks provenance (origin) metadata for memories.
-
-    Singleton per database path. Thread-safe.
-    """
-
-    _instances: Dict[str, "ProvenanceTracker"] = {}
-    _instances_lock = threading.Lock()
-
-    @classmethod
-    def get_instance(cls, db_path: Optional[Path] = None) -> "ProvenanceTracker":
-        """Get or create the singleton ProvenanceTracker."""
-        if db_path is None:
-            db_path = Path.home() / ".claude-memory" / "memory.db"
-        key = str(db_path)
-        with cls._instances_lock:
-            if key not in cls._instances:
-                cls._instances[key] = cls(db_path)
-            return cls._instances[key]
-
-    @classmethod
-    def reset_instance(cls, db_path: Optional[Path] = None) -> None:
-        """Remove singleton. Used for testing."""
-        with cls._instances_lock:
-            if db_path is None:
-                cls._instances.clear()
-            else:
-                key = str(db_path)
-                if key in cls._instances:
-                    del cls._instances[key]
-
-    def __init__(self, db_path: Path):
-        self.db_path = Path(db_path)
-        self._init_schema()
-        logger.info("ProvenanceTracker initialized: db=%s", self.db_path)
-
-    def _init_schema(self):
-        """
-        Add provenance columns to memories table (safe migration).
-
-        Uses ALTER TABLE ADD COLUMN wrapped in try/except — safe for:
-        - Fresh databases (columns don't exist yet)
-        - Existing databases (columns might already exist)
-        - Concurrent migrations (OperationalError caught)
-        """
-        provenance_columns = {
-            'created_by': "TEXT DEFAULT 'user'",
-            'source_protocol': "TEXT DEFAULT 'cli'",
-            'trust_score': "REAL DEFAULT 1.0",
-            'provenance_chain': "TEXT DEFAULT '[]'",
-        }
-
-        try:
-            from db_connection_manager import DbConnectionManager
-            mgr = DbConnectionManager.get_instance(self.db_path)
-
-            def _migrate(conn):
-                cursor = conn.cursor()
-
-                # Check existing columns
-                cursor.execute("PRAGMA table_info(memories)")
-                existing = {row[1] for row in cursor.fetchall()}
-
-                for col_name, col_type in provenance_columns.items():
-                    if col_name not in existing:
-                        try:
-                            cursor.execute(f"ALTER TABLE memories ADD COLUMN {col_name} {col_type}")
-                        except sqlite3.OperationalError:
-                            pass  # Column already exists (concurrent migration)
-
-                # Index for provenance queries
-                try:
-                    cursor.execute("CREATE INDEX IF NOT EXISTS idx_created_by ON memories(created_by)")
-                except sqlite3.OperationalError:
-                    pass
-
-                try:
-                    cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_protocol ON memories(source_protocol)")
-                except sqlite3.OperationalError:
-                    pass
-
-                conn.commit()
-
-            mgr.execute_write(_migrate)
-
-        except ImportError:
-            conn = sqlite3.connect(str(self.db_path))
-            cursor = conn.cursor()
-
-            cursor.execute("PRAGMA table_info(memories)")
-            existing = {row[1] for row in cursor.fetchall()}
-
-            for col_name, col_type in provenance_columns.items():
-                if col_name not in existing:
-                    try:
-                        cursor.execute(f"ALTER TABLE memories ADD COLUMN {col_name} {col_type}")
-                    except sqlite3.OperationalError:
-                        pass
-
-            try:
-                cursor.execute("CREATE INDEX IF NOT EXISTS idx_created_by ON memories(created_by)")
-            except sqlite3.OperationalError:
-                pass
-            try:
-                cursor.execute("CREATE INDEX IF NOT EXISTS idx_source_protocol ON memories(source_protocol)")
-            except sqlite3.OperationalError:
-                pass
-
-            conn.commit()
-            conn.close()
-
-    # =========================================================================
-    # Record Provenance
-    # =========================================================================
-
-    def record_provenance(
-        self,
-        memory_id: int,
-        created_by: str = "user",
-        source_protocol: str = "cli",
-        trust_score: float = 1.0,
-        derived_from: Optional[int] = None,
-    ) -> None:
-        """
-        Record provenance metadata for a memory.
-
-        Called after a memory is created. Updates the provenance columns
-        on the memories table row.
-
-        Args:
-            memory_id: ID of the memory to annotate
-            created_by: Agent ID that created this memory
-            source_protocol: Protocol used (mcp, cli, rest, python)
-            trust_score: Trust score at time of creation
-            derived_from: If this memory was derived from another, its ID
-        """
-        trust_score = max(0.0, min(1.0, trust_score))
-        chain = json.dumps([derived_from] if derived_from else [])
-
-        try:
-            from db_connection_manager import DbConnectionManager
-            mgr = DbConnectionManager.get_instance(self.db_path)
-
-            def _update(conn):
-                conn.execute('''
-                    UPDATE memories
-                    SET created_by = ?, source_protocol = ?,
-                        trust_score = ?, provenance_chain = ?
-                    WHERE id = ?
-                ''', (created_by, source_protocol, trust_score, chain, memory_id))
-                conn.commit()
-
-            mgr.execute_write(_update)
-
-        except Exception as e:
-            # Provenance failure must never break core operations
-            logger.error("Failed to record provenance for memory %d: %s", memory_id, e)
-
-    # =========================================================================
-    # Query Provenance
-    # =========================================================================
-
-    def get_provenance(self, memory_id: int) -> Optional[dict]:
-        """
-        Get provenance metadata for a specific memory.
-
-        Args:
-            memory_id: Memory ID to query
-
-        Returns:
-            Dict with created_by, source_protocol, trust_score, provenance_chain
-            or None if memory not found
-        """
-        try:
-            from db_connection_manager import DbConnectionManager
-            mgr = DbConnectionManager.get_instance(self.db_path)
-
-            with mgr.read_connection() as conn:
-                cursor = conn.cursor()
-                cursor.execute("""
-                    SELECT id, created_by, source_protocol, trust_score, provenance_chain
-                    FROM memories WHERE id = ?
-                """, (memory_id,))
-                row = cursor.fetchone()
-
-            if not row:
-                return None
-
-            chain = []
-            try:
-                chain = json.loads(row[4]) if row[4] else []
-            except (json.JSONDecodeError, TypeError):
-                pass
-
-            return {
-                "memory_id": row[0],
-                "created_by": row[1] or "user",
-                "source_protocol": row[2] or "cli",
-                "trust_score": row[3] if row[3] is not None else 1.0,
-                "provenance_chain": chain,
-            }
-
-        except Exception as e:
-            logger.error("Failed to get provenance for memory %d: %s", memory_id, e)
-            return None
-
-    def get_memories_by_agent(self, agent_id: str, limit: int = 50) -> list:
-        """
-        Get all memories created by a specific agent.
-
-        Args:
-            agent_id: Agent ID to query
-            limit: Max results
-
-        Returns:
-            List of (memory_id, created_at, trust_score) tuples
-        """
-        try:
-            from db_connection_manager import DbConnectionManager
-            mgr = DbConnectionManager.get_instance(self.db_path)
-
-            with mgr.read_connection() as conn:
-                cursor = conn.cursor()
-                cursor.execute("""
-                    SELECT id, created_at, trust_score
-                    FROM memories
-                    WHERE created_by = ?
-                    ORDER BY created_at DESC
-                    LIMIT ?
-                """, (agent_id, limit))
-                return [
-                    {"memory_id": r[0], "created_at": r[1], "trust_score": r[2]}
-                    for r in cursor.fetchall()
-                ]
-
-        except Exception as e:
-            logger.error("Failed to get memories by agent %s: %s", agent_id, e)
-            return []
-
-    def get_provenance_stats(self) -> dict:
-        """Get provenance statistics across all memories."""
-        try:
-            from db_connection_manager import DbConnectionManager
-            mgr = DbConnectionManager.get_instance(self.db_path)
-
-            with mgr.read_connection() as conn:
-                cursor = conn.cursor()
-
-                cursor.execute("""
-                    SELECT created_by, COUNT(*) as count
-                    FROM memories
-                    WHERE created_by IS NOT NULL
-                    GROUP BY created_by
-                    ORDER BY count DESC
-                """)
-                by_agent = dict(cursor.fetchall())
-
-                cursor.execute("""
-                    SELECT source_protocol, COUNT(*) as count
-                    FROM memories
-                    WHERE source_protocol IS NOT NULL
-                    GROUP BY source_protocol
-                    ORDER BY count DESC
-                """)
-                by_protocol = dict(cursor.fetchall())
-
-                cursor.execute("SELECT AVG(trust_score) FROM memories WHERE trust_score IS NOT NULL")
-                avg_trust = cursor.fetchone()[0]
-
-            return {
-                "by_agent": by_agent,
-                "by_protocol": by_protocol,
-                "avg_trust_score": round(avg_trust, 3) if avg_trust else 1.0,
-            }
-
-        except Exception as e:
-            logger.error("Failed to get provenance stats: %s", e)
-            return {"by_agent": {}, "by_protocol": {}, "error": str(e)}