superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/learning/bootstrap.py

@@ -0,0 +1,298 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Synthetic bootstrap for cold-start ML ranking.
+
+Generates (query, fact, label) triples from existing memory patterns
+so the adaptive ranker can operate before real feedback accumulates.
+
+Four strategies:
+  1. Access-based  -- frequently accessed memories are positive for their keywords
+  2. Importance-based -- high-importance memories are positive for their tags
+  3. Pattern-based -- learned identity patterns generate synthetic queries
+  4. Recency-based -- recent memories rank higher for shared-topic queries
+
+Ported from V2 SyntheticBootstrapper with V2-specific deps removed.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import sqlite3
+from collections import Counter
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Stopwords for keyword extraction (no NLP dep)
+# ---------------------------------------------------------------------------
+_STOPWORDS = frozenset({
+    "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+    "have", "has", "had", "do", "does", "did", "will", "would", "shall",
+    "should", "may", "might", "can", "could", "to", "of", "in", "for",
+    "on", "with", "at", "by", "from", "as", "into", "through", "during",
+    "before", "after", "above", "below", "between", "under", "again",
+    "further", "then", "once", "here", "there", "when", "where", "why",
+    "how", "all", "each", "every", "both", "few", "more", "most", "other",
+    "some", "such", "no", "not", "only", "own", "same", "so", "than",
+    "too", "very", "just", "because", "but", "and", "or", "if", "while",
+    "about", "up", "out", "off", "over", "also", "it", "its", "this",
+    "that", "i", "me", "my", "we", "our", "you", "your", "he", "him",
+    "she", "her", "they", "them", "what", "which", "who", "whom",
+})
+
+_WORD_RE = re.compile(r"[a-zA-Z0-9_]+")
+
+
+class SyntheticBootstrap:
+    """Generate synthetic training data for cold-start ML ranking.
+
+    Args:
+        db_path: Path to the sqlite database containing a ``memories`` table.
+                 A fresh database is created if the file does not exist.
+    """
+
+    def __init__(self, db_path: Path | str) -> None:
+        self._db_path = Path(db_path)
+        self._ensure_schema()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def generate(self, profile_id: str, count: int = 200) -> list[dict]:
+        """Generate synthetic (query, fact_id, label) triples.
+
+        Returns up to *count* records.  Returns an empty list when
+        the profile has no memories to mine from.
+        """
+        memories = self._fetch_memories(profile_id)
+        if not memories:
+            return []
+
+        records: list[dict] = []
+        records.extend(self._access_based(memories))
+        records.extend(self._importance_based(memories))
+        records.extend(self._recency_based(memories))
+
+        # Trim to target count with source diversity
+        if len(records) > count:
+            records = self._diverse_sample(records, count)
+
+        return records
+
+    # ------------------------------------------------------------------
+    # Strategy 1: access-based
+    # ------------------------------------------------------------------
+
+    def _access_based(self, memories: list[dict]) -> list[dict]:
+        """Memories accessed 5+ times are positive for their keywords."""
+        records: list[dict] = []
+        high_access = [m for m in memories if m.get("access_count", 0) >= 5]
+
+        for mem in high_access:
+            keywords = _extract_keywords(mem.get("content", ""))
+            if not keywords:
+                continue
+            query = " ".join(keywords)
+
+            records.append(_build_record(query, mem, label=1.0, source="access_pos"))
+
+            # Pick negatives from memories with different tags
+            for neg in self._find_negatives(memories, mem, limit=2):
+                records.append(_build_record(query, neg, label=0.0, source="access_neg"))
+
+        return records
+
+    # ------------------------------------------------------------------
+    # Strategy 2: importance-based
+    # ------------------------------------------------------------------
+
+    def _importance_based(self, memories: list[dict]) -> list[dict]:
+        """High-importance memories (>=8) are positive for their tags."""
+        records: list[dict] = []
+        important = [m for m in memories if (m.get("importance") or 0) >= 8]
+
+        for mem in important:
+            query = self._tags_to_query(mem)
+            if not query:
+                keywords = _extract_keywords(mem.get("content", ""))
+                query = " ".join(keywords) if keywords else ""
+            if not query:
+                continue
+
+            records.append(_build_record(query, mem, label=1.0, source="importance_pos"))
+
+            for neg in self._find_negatives(memories, mem, limit=2):
+                records.append(_build_record(query, neg, label=0.0, source="importance_neg"))
+
+        return records
+
+    # ------------------------------------------------------------------
+    # Strategy 3: recency-based
+    # ------------------------------------------------------------------
+
+    def _recency_based(self, memories: list[dict]) -> list[dict]:
+        """Recent memories rank higher for shared-topic queries."""
+        records: list[dict] = []
+        recent = memories[:30]  # Already sorted by created_at DESC
+        if len(recent) < 4:
+            return records
+
+        seen_queries: set[str] = set()
+        for mem in recent[:15]:
+            keywords = _extract_keywords(mem.get("content", ""))
+            query = " ".join(keywords) if keywords else ""
+            if not query or query in seen_queries:
+                continue
+            seen_queries.add(query)
+
+            records.append(_build_record(query, mem, label=0.8, source="recency_pos"))
+
+            # Older memories about the same topic are weak negatives
+            for older in memories[30:]:
+                content = (older.get("content") or "").lower()
+                if any(kw in content for kw in keywords):
+                    records.append(
+                        _build_record(query, older, label=0.3, source="recency_neg")
+                    )
+                    break
+
+        return records
+
+    # ------------------------------------------------------------------
+    # Database
+    # ------------------------------------------------------------------
+
+    def _ensure_schema(self) -> None:
+        """Create the memories table if it does not exist (for testing)."""
+        conn = sqlite3.connect(str(self._db_path))
+        try:
+            conn.execute(
+                """CREATE TABLE IF NOT EXISTS memories (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    profile_id TEXT DEFAULT 'default',
+                    content TEXT,
+                    tags TEXT,
+                    importance INTEGER DEFAULT 5,
+                    access_count INTEGER DEFAULT 0,
+                    created_at TEXT
+                )"""
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _fetch_memories(self, profile_id: str) -> list[dict]:
+        """Fetch memories for a profile, ordered by recency."""
+        conn = sqlite3.connect(str(self._db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            cur = conn.execute(
+                "SELECT * FROM memories WHERE profile_id = ? "
+                "ORDER BY created_at DESC LIMIT 500",
+                (profile_id,),
+            )
+            return [dict(r) for r in cur.fetchall()]
+        except sqlite3.OperationalError:
+            return []
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _tags_to_query(memory: dict) -> str:
+        """Extract a query string from the memory's tags field."""
+        tags = memory.get("tags", "")
+        if not tags:
+            return ""
+        if isinstance(tags, list):
+            return " ".join(tags[:5])
+        try:
+            parsed = json.loads(tags)
+            if isinstance(parsed, list):
+                return " ".join(str(t) for t in parsed[:5])
+        except (json.JSONDecodeError, TypeError):
+            pass
+        return " ".join(t.strip() for t in str(tags).split(",") if t.strip())[:5]
+
+    @staticmethod
+    def _find_negatives(
+        memories: list[dict], anchor: dict, limit: int = 2
+    ) -> list[dict]:
+        """Pick memories dissimilar to anchor (different tags or id)."""
+        anchor_id = anchor.get("id")
+        anchor_tags = set(
+            t.strip().lower()
+            for t in str(anchor.get("tags", "")).split(",")
+            if t.strip()
+        )
+        negatives: list[dict] = []
+        for mem in memories:
+            if mem.get("id") == anchor_id:
+                continue
+            mem_tags = set(
+                t.strip().lower()
+                for t in str(mem.get("tags", "")).split(",")
+                if t.strip()
+            )
+            if not mem_tags & anchor_tags:
+                negatives.append(mem)
+                if len(negatives) >= limit:
+                    break
+        return negatives
+
+    @staticmethod
+    def _diverse_sample(records: list[dict], target: int) -> list[dict]:
+        """Sample records proportionally across sources."""
+        by_source: dict[str, list[dict]] = {}
+        for r in records:
+            by_source.setdefault(r["source"], []).append(r)
+        n_sources = len(by_source) or 1
+        per_source = max(1, target // n_sources)
+        sampled: list[dict] = []
+        for src_records in by_source.values():
+            sampled.extend(src_records[:per_source])
+        return sampled[:target]
+
+
+# ----------------------------------------------------------------------
+# Module-level helpers (stateless)
+# ----------------------------------------------------------------------
+
+
+def _extract_keywords(content: str, top_n: int = 3) -> list[str]:
+    """Extract top-N keywords from text via frequency (no NLP deps)."""
+    if not content:
+        return []
+    tokens = _WORD_RE.findall(content.lower())
+    filtered = [t for t in tokens if t not in _STOPWORDS and len(t) > 2]
+    counts = Counter(filtered)
+    return [word for word, _ in counts.most_common(top_n)]
+
+
+def _build_record(
+    query: str,
+    memory: dict,
+    label: float,
+    source: str,
+) -> dict:
+    """Build a single training record."""
+    return {
+        "query": query,
+        "query_hash": hashlib.sha256(query.encode()).hexdigest()[:16],
+        "fact_id": memory.get("id", 0),
+        "label": label,
+        "source": source,
+    }

package/src/superlocalmemory/learning/cross_project.py

@@ -0,0 +1,399 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Cross-project knowledge aggregator.
+
+Transfers learned preferences and patterns from completed projects to
+new ones via temporal-decay merging and contradiction detection.
+Ported from V2 CrossProjectAggregator with inline sqlite3 queries.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import re
+import sqlite3
+from collections import Counter
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# Temporal decay half-life in days (1 year)
+DECAY_HALF_LIFE_DAYS = 365.0
+
+# Contradiction detection window in days
+CONTRADICTION_WINDOW_DAYS = 90
+
+# Minimum evidence to consider a pattern valid
+MIN_EVIDENCE_FOR_MERGE = 2
+
+# Minimum confidence for a merged pattern to be stored
+MIN_MERGE_CONFIDENCE = 0.3
+
+# Simple keyword regex for frequency analysis
+_WORD_RE = re.compile(r"[a-zA-Z0-9_]+")
+
+# Technology categories detected from content
+TECH_CATEGORIES: dict[str, list[str]] = {
+    "frontend_framework": [
+        "react", "vue", "angular", "svelte", "next", "nuxt", "solid",
+    ],
+    "backend_framework": [
+        "express", "fastapi", "django", "flask", "spring", "rails", "nest",
+    ],
+    "language": [
+        "python", "typescript", "javascript", "rust", "go", "java", "kotlin",
+    ],
+    "database": [
+        "postgres", "mysql", "sqlite", "mongo", "redis", "dynamodb", "supabase",
+    ],
+    "cloud": ["aws", "azure", "gcp", "vercel", "cloudflare", "netlify"],
+    "testing": ["pytest", "jest", "vitest", "cypress", "playwright"],
+}
+
+# ---------------------------------------------------------------------------
+# Schema for transferable patterns
+# ---------------------------------------------------------------------------
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS transferable_patterns (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    pattern_type TEXT NOT NULL DEFAULT 'preference',
+    key TEXT NOT NULL,
+    value TEXT NOT NULL,
+    confidence REAL DEFAULT 0.0,
+    evidence_count INTEGER DEFAULT 0,
+    profiles_seen INTEGER DEFAULT 1,
+    decay_factor REAL DEFAULT 1.0,
+    contradictions TEXT DEFAULT '[]',
+    first_seen TEXT,
+    last_seen TEXT,
+    UNIQUE(pattern_type, key)
+);
+
+CREATE TABLE IF NOT EXISTS memories (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    profile_id TEXT DEFAULT 'default',
+    content TEXT,
+    project_name TEXT,
+    created_at TEXT
+);
+"""
+
+
+class CrossProjectAggregator:
+    """Aggregate technology preferences across profiles."""
+
+    def __init__(self, db_path: Path | str) -> None:
+        self._db_path = Path(db_path)
+        self._ensure_schema()
+
+    def aggregate(
+        self, source_profiles: list[str], target_profile: str
+    ) -> list[dict]:
+        """Transfer patterns from *source_profiles* to *target_profile*."""
+        if not source_profiles:
+            return []
+
+        # Step 1: Analyse each source profile
+        profile_patterns: list[dict] = []
+        for profile_id in source_profiles:
+            pdata = self._analyse_profile(profile_id)
+            if pdata:
+                profile_patterns.append(pdata)
+
+        if not profile_patterns:
+            return []
+
+        # Step 2: Merge with temporal decay
+        merged = self._merge_with_decay(profile_patterns)
+
+        # Step 3: Detect contradictions
+        for key, pattern_data in merged.items():
+            pattern_data["contradictions"] = self._detect_contradictions(
+                key, pattern_data
+            )
+
+        # Step 4: Store and return
+        self._store_patterns(merged)
+        return [{"key": k, **v} for k, v in merged.items()]
+
+    def get_preferences(self, min_confidence: float = 0.6) -> dict[str, dict]:
+        """Retrieve stored transferable preferences above *min_confidence*."""
+        conn = sqlite3.connect(str(self._db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            cur = conn.execute(
+                "SELECT * FROM transferable_patterns "
+                "WHERE confidence >= ? ORDER BY confidence DESC",
+                (min_confidence,),
+            )
+            result: dict[str, dict] = {}
+            for row in cur.fetchall():
+                d = dict(row)
+                contradictions = _parse_json_list(d.get("contradictions", "[]"))
+                result[d["key"]] = {
+                    "value": d["value"],
+                    "confidence": d["confidence"],
+                    "evidence_count": d["evidence_count"],
+                    "profiles_seen": d.get("profiles_seen", 1),
+                    "decay_factor": d.get("decay_factor", 1.0),
+                    "contradictions": contradictions,
+                }
+            return result
+        except sqlite3.OperationalError:
+            return {}
+        finally:
+            conn.close()
+
+    def _analyse_profile(self, profile_id: str) -> Optional[dict]:
+        """Detect tech category preferences in a single profile."""
+        conn = sqlite3.connect(str(self._db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            cur = conn.execute(
+                "SELECT content, created_at FROM memories "
+                "WHERE profile_id = ? ORDER BY created_at DESC LIMIT 500",
+                (profile_id,),
+            )
+            rows = cur.fetchall()
+        except sqlite3.OperationalError:
+            return None
+        finally:
+            conn.close()
+
+        if not rows:
+            return None
+
+        # Detect tech keywords per category
+        category_counts: dict[str, Counter] = {
+            cat: Counter() for cat in TECH_CATEGORIES
+        }
+        for row in rows:
+            content = (dict(row).get("content") or "").lower()
+            tokens = set(_WORD_RE.findall(content))
+            for category, keywords in TECH_CATEGORIES.items():
+                for kw in keywords:
+                    if kw in tokens:
+                        category_counts[category][kw] += 1
+
+        # Build patterns: winner per category with enough evidence
+        patterns: dict[str, dict] = {}
+        for category, counter in category_counts.items():
+            if not counter:
+                continue
+            winner, win_count = counter.most_common(1)[0]
+            total = sum(counter.values())
+            if win_count >= MIN_EVIDENCE_FOR_MERGE:
+                patterns[category] = {
+                    "value": winner,
+                    "confidence": round(win_count / total, 3) if total else 0.0,
+                    "evidence_count": win_count,
+                }
+
+        if not patterns:
+            return None
+
+        # Latest timestamp for decay calculation
+        latest_ts = dict(rows[0]).get("created_at", datetime.now(UTC).isoformat())
+
+        return {
+            "profile": profile_id,
+            "patterns": patterns,
+            "latest_timestamp": latest_ts,
+            "memory_count": len(rows),
+        }
+
+    def _merge_with_decay(
+        self, profile_patterns: list[dict]
+    ) -> dict[str, dict]:
+        now = datetime.now(UTC)
+        contributions: dict[str, list[dict]] = {}
+
+        for pdata in profile_patterns:
+            age_days = _days_since(pdata["latest_timestamp"], now)
+            weight = math.exp(-age_days / DECAY_HALF_LIFE_DAYS)
+
+            for cat_key, pattern in pdata["patterns"].items():
+                contributions.setdefault(cat_key, []).append({
+                    "value": pattern["value"],
+                    "confidence": pattern["confidence"],
+                    "evidence_count": pattern["evidence_count"],
+                    "weight": weight,
+                    "profile": pdata["profile"],
+                    "latest_timestamp": pdata["latest_timestamp"],
+                })
+
+        merged: dict[str, dict] = {}
+        for cat_key, contribs in contributions.items():
+            result = self._merge_single(contribs)
+            if result is not None:
+                merged[cat_key] = result
+        return merged
+
+    @staticmethod
+    def _merge_single(contributions: list[dict]) -> Optional[dict]:
+        """Merge contributions for one category across profiles."""
+        value_scores: dict[str, float] = {}
+        value_evidence: dict[str, int] = {}
+        value_profiles: dict[str, set] = {}
+        total_weighted = 0.0
+
+        for c in contributions:
+            v = c["value"]
+            w_ev = c["evidence_count"] * c["weight"]
+            value_scores[v] = value_scores.get(v, 0.0) + w_ev
+            value_evidence[v] = value_evidence.get(v, 0) + c["evidence_count"]
+            value_profiles.setdefault(v, set()).add(c["profile"])
+            total_weighted += w_ev
+
+        if total_weighted == 0:
+            return None
+
+        winner = max(value_scores, key=lambda k: value_scores[k])
+        confidence = value_scores[winner] / total_weighted
+        ev = value_evidence[winner]
+
+        if ev < MIN_EVIDENCE_FOR_MERGE or confidence < MIN_MERGE_CONFIDENCE:
+            return None
+
+        return {
+            "value": winner,
+            "confidence": round(min(0.95, confidence), 3),
+            "evidence_count": ev,
+            "profiles_seen": len(value_profiles[winner]),
+            "decay_factor": round(
+                max(c["weight"] for c in contributions if c["value"] == winner),
+                4,
+            ),
+            "contradictions": [],
+        }
+
+    def _detect_contradictions(
+        self, pattern_key: str, pattern_data: dict
+    ) -> list[str]:
+        contradictions: list[str] = []
+
+        # Check stored value vs new value
+        conn = sqlite3.connect(str(self._db_path))
+        conn.row_factory = sqlite3.Row
+        try:
+            cur = conn.execute(
+                "SELECT value, last_seen FROM transferable_patterns "
+                "WHERE key = ? AND pattern_type = 'preference'",
+                (pattern_key,),
+            )
+            row = cur.fetchone()
+            if row:
+                d = dict(row)
+                old_val = d.get("value", "")
+                if old_val and old_val != pattern_data["value"]:
+                    old_ts = d.get("last_seen", "")
+                    if old_ts and _is_within_window(old_ts, CONTRADICTION_WINDOW_DAYS):
+                        contradictions.append(
+                            f"Changed from '{old_val}' to "
+                            f"'{pattern_data['value']}' within "
+                            f"{CONTRADICTION_WINDOW_DAYS} days"
+                        )
+        except sqlite3.OperationalError:
+            pass
+        finally:
+            conn.close()
+
+        return contradictions
+
+    def _ensure_schema(self) -> None:
+        conn = sqlite3.connect(str(self._db_path))
+        try:
+            conn.executescript(_SCHEMA)
+        finally:
+            conn.close()
+
+    def _store_patterns(self, merged: dict[str, dict]) -> None:
+        conn = sqlite3.connect(str(self._db_path))
+        now = datetime.now(UTC).isoformat()
+        try:
+            for key, data in merged.items():
+                conn.execute(
+                    """INSERT INTO transferable_patterns
+                       (pattern_type, key, value, confidence, evidence_count,
+                        profiles_seen, decay_factor, contradictions,
+                        first_seen, last_seen)
+                       VALUES ('preference', ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                       ON CONFLICT(pattern_type, key) DO UPDATE SET
+                           value = excluded.value,
+                           confidence = excluded.confidence,
+                           evidence_count = excluded.evidence_count,
+                           profiles_seen = excluded.profiles_seen,
+                           decay_factor = excluded.decay_factor,
+                           contradictions = excluded.contradictions,
+                           last_seen = excluded.last_seen
+                    """,
+                    (
+                        key,
+                        data["value"],
+                        data["confidence"],
+                        data["evidence_count"],
+                        data.get("profiles_seen", 1),
+                        data.get("decay_factor", 1.0),
+                        json.dumps(data.get("contradictions", [])),
+                        now,
+                        now,
+                    ),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+
+
+# ----------------------------------------------------------------------
+# Module-level helpers
+# ----------------------------------------------------------------------
+
+
+def _parse_json_list(raw: Any) -> list:
+    """Parse a JSON string into a list, returning [] on failure."""
+    if isinstance(raw, list):
+        return raw
+    if isinstance(raw, str):
+        try:
+            return json.loads(raw)
+        except (json.JSONDecodeError, TypeError):
+            pass
+    return []
+
+
+def _days_since(timestamp_str: str, now: datetime | None = None) -> float:
+    if now is None:
+        now = datetime.now(UTC)
+    if not timestamp_str:
+        return 0.0
+    try:
+        ts = datetime.fromisoformat(str(timestamp_str).replace(" ", "T"))
+        return max(0.0, (now - ts).total_seconds() / 86400.0)
+    except (ValueError, AttributeError, TypeError):
+        pass
+    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S.%f"):
+        try:
+            ts = datetime.strptime(str(timestamp_str), fmt)
+            return max(0.0, (now - ts).total_seconds() / 86400.0)
+        except (ValueError, TypeError):
+            continue
+    return 0.0
+
+
+def _is_within_window(timestamp_str: str, window_days: int) -> bool:
+    if not timestamp_str:
+        return False
+    try:
+        ts = datetime.fromisoformat(str(timestamp_str).replace(" ", "T"))
+        return (datetime.now(UTC) - ts).days <= window_days
+    except (ValueError, AttributeError, TypeError):
+        return False