superlocalmemory 2.7.6 → 2.8.0

package/src/patterns/scoring.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Confidence Scoring - Bayesian pattern confidence calculation.
+
+Uses Beta-Binomial posterior with log-scaled competition,
+recency bonuses, and temporal distribution factors.
+"""
+
+import sqlite3
+import logging
+from datetime import datetime, timedelta
+from typing import List
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class ConfidenceScorer:
+    """Calculates and tracks pattern confidence scores."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+
+    def calculate_confidence(
+        self,
+        pattern_type: str,
+        key: str,
+        value: str,
+        evidence_memory_ids: List[int],
+        total_memories: int
+    ) -> float:
+        """
+        Calculate confidence using Beta-Binomial Bayesian posterior.
+
+        Based on MACLA (arXiv:2512.18950, Forouzandeh et al., Dec 2025):
+          posterior_mean = (alpha + evidence) / (alpha + beta + evidence + competition)
+
+        Adaptation: MACLA's Beta-Binomial uses pairwise interaction counts.
+        Our corpus has sparse signals (most memories are irrelevant to any
+        single pattern). We use log-scaled competition instead of raw total
+        to avoid over-dilution: competition = log2(total_memories).
+
+        Pattern-specific priors (alpha, beta):
+        - preference (1, 4): prior mean 0.20, ~8 items to reach 0.5
+        - style (1, 5): prior mean 0.17, subtler signals need more evidence
+        - terminology (2, 3): prior mean 0.40, direct usage signal
+        """
+        if total_memories == 0 or not evidence_memory_ids:
+            return 0.0
+
+        import math
+        evidence_count = len(evidence_memory_ids)
+
+        # Pattern-specific Beta priors (alpha, beta)
+        PRIORS = {
+            'preference': (1.0, 4.0),
+            'style':      (1.0, 5.0),
+            'terminology': (2.0, 3.0),
+        }
+        alpha, beta = PRIORS.get(pattern_type, (1.0, 4.0))
+
+        # Log-scaled competition: grows slowly with corpus size
+        # 10 memories -> 3.3, 60 -> 5.9, 500 -> 9.0, 5000 -> 12.3
+        competition = math.log2(max(2, total_memories))
+
+        # MACLA-inspired Beta posterior with log competition
+        posterior_mean = (alpha + evidence_count) / (alpha + beta + evidence_count + competition)
+
+        # Recency adjustment (mild: 1.0 to 1.15)
+        recency_bonus = self._calculate_recency_bonus(evidence_memory_ids)
+        recency_factor = 1.0 + min(0.15, 0.075 * (recency_bonus - 1.0) / 0.2) if recency_bonus > 1.0 else 1.0
+
+        # Temporal spread adjustment (0.9 to 1.1)
+        distribution_factor = self._calculate_distribution_factor(evidence_memory_ids)
+
+        # Final confidence
+        confidence = posterior_mean * recency_factor * distribution_factor
+
+        return min(0.95, round(confidence, 3))
+
+    def _calculate_recency_bonus(self, memory_ids: List[int]) -> float:
+        """Give bonus to patterns with recent evidence."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        # Get timestamps
+        placeholders = ','.join('?' * len(memory_ids))
+        cursor.execute(f'''
+            SELECT created_at FROM memories
+            WHERE id IN ({placeholders})
+            ORDER BY created_at DESC
+        ''', memory_ids)
+
+        timestamps = cursor.fetchall()
+        conn.close()
+
+        if not timestamps:
+            return 1.0
+
+        # Check if any memories are from last 30 days
+        recent_count = 0
+        cutoff = datetime.now() - timedelta(days=30)
+
+        for ts_tuple in timestamps:
+            ts_str = ts_tuple[0]
+            try:
+                ts = datetime.fromisoformat(ts_str.replace(' ', 'T'))
+                if ts > cutoff:
+                    recent_count += 1
+            except (ValueError, AttributeError):
+                pass
+
+        # Bonus if >50% are recent
+        if len(timestamps) > 0 and recent_count / len(timestamps) > 0.5:
+            return 1.2
+        else:
+            return 1.0
+
+    def _calculate_distribution_factor(self, memory_ids: List[int]) -> float:
+        """Better confidence if memories are distributed over time, not just one session."""
+        if len(memory_ids) < 3:
+            return 0.8  # Penalize low sample size
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        placeholders = ','.join('?' * len(memory_ids))
+        cursor.execute(f'''
+            SELECT created_at FROM memories
+            WHERE id IN ({placeholders})
+            ORDER BY created_at
+        ''', memory_ids)
+
+        timestamps = [row[0] for row in cursor.fetchall()]
+        conn.close()
+
+        if len(timestamps) < 2:
+            return 0.8
+
+        try:
+            # Parse timestamps
+            dates = []
+            for ts_str in timestamps:
+                try:
+                    ts = datetime.fromisoformat(ts_str.replace(' ', 'T'))
+                    dates.append(ts)
+                except (ValueError, AttributeError):
+                    pass
+
+            if len(dates) < 2:
+                return 0.8
+
+            # Calculate time span
+            time_span = (dates[-1] - dates[0]).days
+
+            # If memories span multiple days, higher confidence
+            if time_span > 7:
+                return 1.1
+            elif time_span > 1:
+                return 1.0
+            else:
+                return 0.9  # All on same day = might be one-off
+
+        except Exception:
+            return 1.0

package/src/patterns/store.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Pattern Store - SQLite-backed pattern storage and retrieval.
+
+Handles identity_patterns and pattern_examples tables,
+including schema migration, CRUD operations, and profile-scoped queries.
+"""
+
+import sqlite3
+import json
+import logging
+from typing import Dict, List, Optional, Any
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class PatternStore:
+    """Handles pattern storage and retrieval."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+        self._init_tables()
+
+    def _init_tables(self):
+        """Initialize pattern tables if they don't exist, or recreate if schema is incomplete."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        # Check if existing tables have correct schema
+        for table_name, required_cols in [
+            ('identity_patterns', {'pattern_type', 'key', 'value', 'confidence'}),
+            ('pattern_examples', {'pattern_id', 'memory_id'}),
+        ]:
+            cursor.execute(f"PRAGMA table_info({table_name})")
+            existing_cols = {row[1] for row in cursor.fetchall()}
+            if existing_cols and not required_cols.issubset(existing_cols):
+                logger.warning(f"Dropping incomplete {table_name} table (missing: {required_cols - existing_cols})")
+                cursor.execute(f'DROP TABLE IF EXISTS {table_name}')
+
+        # Identity patterns table
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS identity_patterns (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                pattern_type TEXT NOT NULL,
+                key TEXT NOT NULL,
+                value TEXT NOT NULL,
+                confidence REAL DEFAULT 0.5,
+                evidence_count INTEGER DEFAULT 1,
+                memory_ids TEXT,
+                category TEXT,
+                profile TEXT DEFAULT 'default',
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                UNIQUE(pattern_type, key, category, profile)
+            )
+        ''')
+
+        # Add profile column if upgrading from older schema
+        try:
+            cursor.execute('ALTER TABLE identity_patterns ADD COLUMN profile TEXT DEFAULT "default"')
+        except sqlite3.OperationalError:
+            pass  # Column already exists
+
+        # Pattern examples table
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS pattern_examples (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                pattern_id INTEGER NOT NULL,
+                memory_id INTEGER NOT NULL,
+                example_text TEXT,
+                FOREIGN KEY (pattern_id) REFERENCES identity_patterns(id) ON DELETE CASCADE,
+                FOREIGN KEY (memory_id) REFERENCES memories(id) ON DELETE CASCADE
+            )
+        ''')
+
+        # Indexes
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_type ON identity_patterns(pattern_type)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_confidence ON identity_patterns(confidence)')
+        cursor.execute('CREATE INDEX IF NOT EXISTS idx_pattern_profile ON identity_patterns(profile)')
+
+        conn.commit()
+        conn.close()
+
+    def save_pattern(self, pattern: Dict[str, Any]) -> int:
+        """Save or update a pattern (scoped by profile)."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        profile = pattern.get('profile', 'default')
+
+        try:
+            # Check if pattern exists for this profile
+            cursor.execute('''
+                SELECT id FROM identity_patterns
+                WHERE pattern_type = ? AND key = ? AND category = ? AND profile = ?
+            ''', (pattern['pattern_type'], pattern['key'], pattern['category'], profile))
+
+            existing = cursor.fetchone()
+
+            memory_ids_json = json.dumps(pattern['memory_ids'])
+
+            if existing:
+                # Update existing pattern
+                pattern_id = existing[0]
+                cursor.execute('''
+                    UPDATE identity_patterns
+                    SET value = ?, confidence = ?, evidence_count = ?,
+                        memory_ids = ?, updated_at = CURRENT_TIMESTAMP
+                    WHERE id = ?
+                ''', (
+                    pattern['value'],
+                    pattern['confidence'],
+                    pattern['evidence_count'],
+                    memory_ids_json,
+                    pattern_id
+                ))
+            else:
+                # Insert new pattern
+                cursor.execute('''
+                    INSERT INTO identity_patterns
+                    (pattern_type, key, value, confidence, evidence_count, memory_ids, category, profile)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ''', (
+                    pattern['pattern_type'],
+                    pattern['key'],
+                    pattern['value'],
+                    pattern['confidence'],
+                    pattern['evidence_count'],
+                    memory_ids_json,
+                    pattern['category'],
+                    profile
+                ))
+                pattern_id = cursor.lastrowid
+
+            # Save examples
+            self._save_pattern_examples(cursor, pattern_id, pattern['memory_ids'], pattern['key'])
+
+            conn.commit()
+            return pattern_id
+
+        finally:
+            conn.close()
+
+    def _save_pattern_examples(self, cursor, pattern_id: int, memory_ids: List[int], key: str):
+        """Save representative examples for pattern."""
+        # Clear old examples
+        cursor.execute('DELETE FROM pattern_examples WHERE pattern_id = ?', (pattern_id,))
+
+        # Save top 3 examples
+        for memory_id in memory_ids[:3]:
+            cursor.execute('SELECT content FROM memories WHERE id = ?', (memory_id,))
+            row = cursor.fetchone()
+
+            if row:
+                content = row[0]
+                excerpt = self._extract_relevant_excerpt(content, key)
+
+                cursor.execute('''
+                    INSERT INTO pattern_examples (pattern_id, memory_id, example_text)
+                    VALUES (?, ?, ?)
+                ''', (pattern_id, memory_id, excerpt))
+
+    def _extract_relevant_excerpt(self, content: str, key: str) -> str:
+        """Extract 150-char excerpt showing pattern."""
+        # Find first mention of key term
+        key_lower = key.lower().replace('_', ' ')
+        idx = content.lower().find(key_lower)
+
+        if idx >= 0:
+            start = max(0, idx - 50)
+            end = min(len(content), idx + 100)
+            excerpt = content[start:end]
+            return excerpt if len(excerpt) <= 150 else excerpt[:150] + '...'
+
+        # Fallback: first 150 chars
+        return content[:150] + ('...' if len(content) > 150 else '')
+
+    def get_patterns(self, min_confidence: float = 0.7, pattern_type: Optional[str] = None,
+                     profile: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Get patterns above confidence threshold, optionally filtered by profile."""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        # Build query with optional filters
+        conditions = ['confidence >= ?']
+        params = [min_confidence]
+
+        if pattern_type:
+            conditions.append('pattern_type = ?')
+            params.append(pattern_type)
+
+        if profile:
+            conditions.append('profile = ?')
+            params.append(profile)
+
+        where_clause = ' AND '.join(conditions)
+        cursor.execute(f'''
+            SELECT id, pattern_type, key, value, confidence, evidence_count,
+                   updated_at, created_at, category
+            FROM identity_patterns
+            WHERE {where_clause}
+            ORDER BY confidence DESC, evidence_count DESC
+        ''', params)
+
+        patterns = []
+        for row in cursor.fetchall():
+            patterns.append({
+                'id': row[0],
+                'pattern_type': row[1],
+                'key': row[2],
+                'value': row[3],
+                'confidence': row[4],
+                'evidence_count': row[5],
+                'frequency': row[5],
+                'last_seen': row[6],
+                'created_at': row[7],
+                'category': row[8]
+            })
+
+        conn.close()
+        return patterns

package/src/patterns/terminology.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""
+Terminology Learner - User-specific term definition extraction.
+
+Learns how the user defines ambiguous terms like 'optimize', 'refactor', etc.
+by analyzing contextual co-occurrence patterns across memories.
+"""
+
+import sqlite3
+import re
+import logging
+from typing import Dict, List, Optional, Any
+from collections import Counter
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+class TerminologyLearner:
+    """Learns user-specific definitions of common terms."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = db_path
+
+        # Common ambiguous terms to learn
+        self.ambiguous_terms = [
+            'optimize', 'refactor', 'clean', 'simple',
+            'mvp', 'prototype', 'scale', 'production-ready',
+            'fix', 'improve', 'update', 'enhance'
+        ]
+
+    def learn_terminology(self, memory_ids: List[int]) -> Dict[str, Dict[str, Any]]:
+        """Learn user-specific term definitions."""
+        patterns = {}
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        for term in self.ambiguous_terms:
+            contexts = []
+
+            # Find all contexts where term appears
+            for memory_id in memory_ids:
+                cursor.execute('SELECT content FROM memories WHERE id = ?', (memory_id,))
+                row = cursor.fetchone()
+
+                if not row:
+                    continue
+
+                content = row[0]
+
+                # Find term in content (case-insensitive)
+                pattern = r'\b' + re.escape(term) + r'\b'
+                for match in re.finditer(pattern, content, re.IGNORECASE):
+                    term_idx = match.start()
+
+                    # Extract 100-char window around term
+                    start = max(0, term_idx - 100)
+                    end = min(len(content), term_idx + len(term) + 100)
+                    context_window = content[start:end]
+
+                    contexts.append({
+                        'memory_id': memory_id,
+                        'context': context_window
+                    })
+
+            # Analyze contexts to extract meaning (need at least 3 examples)
+            if len(contexts) >= 3:
+                definition = self._extract_definition(term, contexts)
+
+                if definition:
+                    evidence_list = list(set([ctx['memory_id'] for ctx in contexts]))
+
+                    # Confidence increases with more examples, capped at 0.95
+                    confidence = min(0.95, 0.6 + (len(contexts) * 0.05))
+
+                    patterns[term] = {
+                        'pattern_type': 'terminology',
+                        'key': term,
+                        'value': definition,
+                        'confidence': round(confidence, 2),
+                        'evidence_count': len(evidence_list),
+                        'memory_ids': evidence_list,
+                        'category': 'general'
+                    }
+
+        conn.close()
+        return patterns
+
+    def _extract_definition(self, term: str, contexts: List[Dict]) -> Optional[str]:
+        """Extract definition from contexts using pattern matching."""
+        # Collect words near the term across all contexts
+        nearby_words = []
+
+        for ctx in contexts:
+            words = re.findall(r'\b\w+\b', ctx['context'].lower())
+            nearby_words.extend(words)
+
+        # Count word frequencies
+        word_counts = Counter(nearby_words)
+
+        # Remove the term itself and common stopwords
+        stopwords = {'the', 'a', 'an', 'is', 'to', 'for', 'of', 'in', 'on', 'at',
+                     'and', 'or', 'but', 'with', 'from', 'by', 'this', 'that'}
+        word_counts = Counter({w: c for w, c in word_counts.items()
+                              if w not in stopwords and w != term.lower()})
+
+        # Get top co-occurring words
+        top_words = [w for w, _ in word_counts.most_common(8)]
+
+        # Apply heuristic rules based on term and context
+        if term == 'optimize':
+            if any(w in top_words for w in ['performance', 'speed', 'faster', 'latency']):
+                return "Performance optimization (speed/latency)"
+            elif any(w in top_words for w in ['code', 'clean', 'refactor']):
+                return "Code quality optimization"
+
+        elif term == 'refactor':
+            if any(w in top_words for w in ['architecture', 'structure', 'design']):
+                return "Architecture change, not just renaming"
+            elif any(w in top_words for w in ['clean', 'organize', 'simplify']):
+                return "Code organization improvement"
+
+        elif term == 'mvp':
+            if any(w in top_words for w in ['core', 'basic', 'essential', 'minimal']):
+                return "Core features only, no polish"
+
+        elif term == 'production-ready':
+            if any(w in top_words for w in ['test', 'error', 'monitoring', 'deploy']):
+                return "Fully tested and monitored for deployment"
+
+        # Generic definition if specific pattern not matched
+        if len(top_words) >= 3:
+            return f"Commonly used with: {', '.join(top_words[:3])}"
+
+        return None

package/src/provenance_tracker.py

@@ -1,22 +1,12 @@
 #!/usr/bin/env python3
-"""
-SuperLocalMemory V2 - Provenance Tracker
-Copyright (c) 2026 Varun Pratap Bhardwaj
-Licensed under MIT License
-
-Repository: https://github.com/varun369/SuperLocalMemoryV2
-Author: Varun Pratap Bhardwaj (Solution Architect)
-
-NOTICE: This software is protected by MIT License.
-Attribution must be preserved in all copies or derivatives.
-"""
-
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
 """
 ProvenanceTracker — Tracks the origin and lineage of every memory.
 
 Adds provenance columns to the memories table:
     created_by       — Agent ID that created this memory (e.g., "mcp:claude-desktop")
-    source_protocol  — Protocol used (mcp, cli, rest, python, a2a)
+    source_protocol  — Protocol used (mcp, cli, rest, python)
     trust_score      — Trust score at time of creation (default 1.0)
     provenance_chain — JSON array of derivation history
 
@@ -174,7 +164,7 @@ class ProvenanceTracker:
         Args:
             memory_id: ID of the memory to annotate
             created_by: Agent ID that created this memory
-            source_protocol: Protocol used (mcp, cli, rest, python, a2a)
+            source_protocol: Protocol used (mcp, cli, rest, python)
             trust_score: Trust score at time of creation
             derived_from: If this memory was derived from another, its ID
         """

package/src/query_optimizer.py

@@ -1,16 +1,14 @@
 #!/usr/bin/env python3
-"""
-SuperLocalMemory V2 - Query Optimizer
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""SuperLocalMemory V2 - Query Optimizer
 
-Copyright (c) 2026 Varun Pratap Bhardwaj
 Solution Architect & Original Creator
 
-Licensed under MIT License (see LICENSE file)
-Repository: https://github.com/varun369/SuperLocalMemoryV2
+ (see LICENSE file)
 
 ATTRIBUTION REQUIRED: This notice must be preserved in all copies.
 """
-
 """
 Query Optimizer - Query Enhancement and Rewriting
 

package/src/rate_limiter.py

@@ -1,10 +1,6 @@
 #!/usr/bin/env python3
-"""
-SuperLocalMemory V2 - Rate Limiter
-Copyright (c) 2026 Varun Pratap Bhardwaj
-Licensed under MIT License
-"""
-
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
 """
 Lightweight rate limiter using sliding window algorithm.
 Pure stdlib — no external dependencies.

package/src/search/__init__.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 SuperLocalMemory (superlocalmemory.com)
+"""search package - Hybrid Search System for SuperLocalMemory V2
+
+Re-exports all public classes so that
+``from search import HybridSearchEngine`` works.
+"""
+from search.engine import HybridSearchEngine
+from search.fusion import FusionMixin
+from search.methods import SearchMethodsMixin
+from search.index_loader import IndexLoaderMixin
+from search.cli import main
+
+__all__ = [
+    "HybridSearchEngine",
+    "FusionMixin",
+    "SearchMethodsMixin",
+    "IndexLoaderMixin",
+    "main",
+]