superlocalmemory 3.4.9 → 3.4.11

package/src/superlocalmemory/encoding/entity_resolver.py

@@ -52,6 +52,9 @@ _PLACE_MARKERS = ("City", "State", "County", "Island", "River", "Mountain",
                   "Lake", "Park", "Street", "Avenue", "Road", "District")
 _EVENT_MARKERS = ("Festival", "Conference", "Summit", "Workshop", "Meeting",
                   "Election", "War", "Match", "Game", "Concert", "Wedding")
+# v3.4.10: Skill entity type — skills, commands, agents, plugins
+_SKILL_MARKERS_RE = re.compile(r'\b(?:skill|command|agent|plugin|hook|mcp)\b', re.IGNORECASE)
+_SKILL_NAMESPACE_RE = re.compile(r"^[\w-]+:[\w-]+$")  # e.g., "superpowers:brainstorming"
 
 
 # ---------------------------------------------------------------------------
@@ -113,25 +116,67 @@ def jaro_winkler(s1: str, s2: str, prefix_weight: float = 0.1) -> float:
 
 
 _COMMON_WORDS = frozenset({
-    "april", "may", "june", "march", "august", "phase", "test", "gap",
-    "dashboard", "remaining", "session", "results", "tools", "projects",
-    "prompts", "integration", "cli", "engagement", "mode", "error",
-    "step", "fix", "build", "check", "run", "start", "stop", "config",
-    "status", "version", "query", "data", "file", "path", "node", "edge",
-    "table", "index", "schema", "model", "type", "class", "function",
-    "module", "package", "import", "export", "default", "pattern",
+    # Months / time words (biggest source of garbage entities)
+    "january", "february", "march", "april", "may", "june", "july",
+    "august", "september", "october", "november", "december",
+    "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
+    "today", "tomorrow", "yesterday", "morning", "evening", "night",
+    # English stop words that get capitalized at sentence start
+    "a", "an", "the", "all", "not", "no", "yes", "and", "or", "but",
+    "if", "is", "are", "was", "were", "be", "been", "being", "have",
+    "has", "had", "do", "does", "did", "will", "would", "shall", "should",
+    "can", "could", "just", "also", "only", "very", "too", "so", "then",
+    "than", "that", "this", "these", "those", "here", "there", "where",
+    "when", "what", "which", "who", "whom", "how", "why", "each", "every",
+    "both", "few", "more", "most", "other", "some", "such", "any", "many",
+    "much", "own", "same", "new", "old", "first", "last", "next", "now",
+    "still", "already", "always", "never", "often", "sometimes", "about",
+    "above", "after", "again", "against", "along", "among", "around",
+    "before", "below", "between", "beyond", "during", "except", "from",
+    "into", "near", "off", "onto", "out", "over", "past", "since",
+    "through", "under", "until", "upon", "with", "within", "without",
+    # Technical stop words (common in dev sessions)
+    "phase", "test", "gap", "dashboard", "remaining", "session", "results",
+    "tools", "projects", "prompts", "integration", "cli", "engagement",
+    "mode", "error", "step", "fix", "build", "check", "run", "start",
+    "stop", "config", "status", "version", "query", "data", "file", "path",
+    "node", "edge", "table", "index", "schema", "model", "type", "class",
+    "function", "module", "package", "import", "export", "default", "pattern",
     "memory", "profile", "context", "pipeline", "worker", "daemon",
     "server", "client", "route", "endpoint", "handler", "hook",
+    "feature", "release", "update", "upgrade", "deploy", "debug", "log",
+    "output", "input", "key", "value", "true", "false", "null", "none",
+    "ready", "done", "todo", "complete", "pending", "active", "failed",
+    "success", "warning", "critical", "high", "medium", "low",
+    "total", "count", "list", "item", "entry", "record", "row", "column",
+    "source", "target", "origin", "destination", "backup", "restore",
+    "create", "read", "delete", "remove", "add", "set", "get", "put",
+    "push", "pull", "fetch", "send", "receive", "request", "response",
+    "enable", "disable", "open", "close", "load", "save", "reset",
+    # Abstract nouns often misclassified as people
+    "completeness", "correctness", "limitations", "requirements",
+    "dependencies", "performance", "security", "quality", "coverage",
+    "progress", "analysis", "research", "implementation", "verification",
+    "overview", "summary", "details", "notes", "changes", "issues",
+    "approach", "strategy", "solution", "problem", "question", "answer",
 })
 
 
 def _guess_entity_type(name: str) -> str:
     """Heuristic entity type classification from name string.
 
-    v3.4.8: Fixed false-positive "person" classification. Single capitalized
-    common words (April, Phase, Dashboard) are concepts, not people.
-    Only classify as "person" when it looks like a real human name.
+    v3.4.10: Aggressive false-positive prevention. "person" is assigned ONLY
+    when the name looks like a real human name (2-3 capitalized words, none
+    in the stop list). Everything else defaults to "concept".
     """
+    # Reject very short or very long names
+    if len(name) <= 2 or len(name) > 100:
+        return "concept"
+
+    # Reject pure numbers, dates, version strings
+    if re.match(r"^[\d.v\-/]+$", name):
+        return "concept"
+
     if any(m in name for m in _ORG_MARKERS):
         return "organization"
     if any(m in name for m in _PLACE_MARKERS):
@@ -139,32 +184,38 @@ def _guess_entity_type(name: str) -> str:
     if any(m in name for m in _EVENT_MARKERS):
         return "event"
 
-    # Filter out common words that aren't people
-    if name.lower() in _COMMON_WORDS:
+    # v3.4.10: Skill entities — namespaced skills always skill type
+    if _SKILL_NAMESPACE_RE.match(name):
+        return "skill"
+
+    # Check ALL words against the stop list (not just the full name)
+    words = name.lower().split()
+    if any(w in _COMMON_WORDS for w in words):
         return "concept"
 
-    # Two capitalized words = likely a person name (e.g. "Varun Bhardwaj")
-    if re.match(r"^[A-Z][a-z]+ [A-Z][a-z]+$", name):
-        # But not if either word is a common term
-        parts = name.lower().split()
-        if not any(p in _COMMON_WORDS for p in parts):
+    # v3.4.10: Skill entities — word-boundary match AFTER common-word filter
+    if _SKILL_MARKERS_RE.search(name):
+        return "skill"
+
+    # Multi-word entity: "person" only if 2-3 capitalized words, no stop words
+    if re.match(r"^[A-Z][a-z]+ [A-Z][a-z]+( [A-Z][a-z]+)?$", name):
+        if not any(p in _COMMON_WORDS for p in words):
             return "person"
 
-    # Single short capitalized word with no digits or dots = concept, not person
-    # "person" should only be assigned for real names, not generic terms
+    # Single capitalized word — almost never a person in our context
+    # Only known first names should get "person" but we can't maintain
+    # a name dictionary, so default to "concept"
     if re.match(r"^[A-Z][a-z]+$", name):
-        if name.lower() in _COMMON_WORDS:
-            return "concept"
-        # Only classify as person if it's a plausible first name
-        # (short word not in common terms — still a heuristic)
-        if len(name) <= 3:
-            return "concept"
-        return "person"
-
-    # Contains dots/slashes/hyphens = likely a technical term
+        return "concept"
+
+    # Contains dots/slashes/hyphens/underscores = technical term
     if re.search(r"[./\-_]", name):
         return "concept"
 
+    # ALL-CAPS or mixed case with numbers = technical/concept
+    if re.match(r"^[A-Z]+$", name) or re.search(r"\d", name):
+        return "concept"
+
     return "concept"
 
 
@@ -211,6 +262,23 @@ class EntityResolver:
             if not name or name.lower() in PRONOUNS:
                 continue
 
+            # Skip very short/long entities
+            if len(name) <= 2 or len(name) > 100:
+                continue
+
+            # Skip single-word stop words
+            words = name.lower().split()
+            if len(words) == 1 and name.lower() in _COMMON_WORDS:
+                continue
+
+            # Skip multi-word entities where ALL words are stop words or <=2 chars
+            if len(words) > 1 and all(w in _COMMON_WORDS or len(w) <= 2 for w in words):
+                continue
+
+            # Skip pure numbers/versions
+            if re.match(r"^[\d.v\-/]+$", name):
+                continue
+
             # Tier a: exact match on canonical_name
             entity = self._db.get_entity_by_name(name, profile_id)
             if entity is not None:

package/src/superlocalmemory/evolution/__init__.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Skill Evolution Engine — track, analyze, and evolve AI agent skills.
+
+3-trigger system (post-session + degradation + health check) with
+LLM confirmation gate and blind verification.
+
+Inspired by: HKUDS/OpenSpace (arXiv:2604.01687), ECC continuous learning.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from superlocalmemory.evolution.types import (
+    EvolutionCandidate,
+    EvolutionRecord,
+    EvolutionType,
+    TriggerType,
+    EvolutionStatus,
+)
+
+__all__ = [
+    "EvolutionCandidate",
+    "EvolutionRecord",
+    "EvolutionType",
+    "TriggerType",
+    "EvolutionStatus",
+]

package/src/superlocalmemory/evolution/blind_verifier.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Blind Verifier — information-isolated skill verification.
+
+The key insight from EvoSkills (arXiv:2604.01687): when a generator
+creates a skill and the same model verifies it, confirmation bias is
+nearly guaranteed. The verifier must be BLIND to the generator's reasoning.
+
+This verifier:
+- Uses a DIFFERENT model from the generator (Haiku vs Sonnet)
+- CANNOT see: original skill, mutation rationale, generator's reasoning
+- CAN see: task description (what the skill should do), evolved SKILL.md
+- Evaluates independently: "Does this skill correctly address the task?"
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class VerificationResult:
+    """Result of blind verification."""
+    passed: bool
+    confidence: float  # 0.0-1.0
+    issues: tuple[str, ...] = ()
+    reasoning: str = ""
+
+
+def build_verification_prompt(
+    skill_name: str,
+    skill_description: str,
+    evolved_content: str,
+) -> str:
+    """Build blind verification prompt.
+
+    The verifier sees ONLY:
+    - What the skill is supposed to do (name + description)
+    - The evolved skill content
+
+    The verifier does NOT see:
+    - The original skill
+    - Why it was evolved
+    - What evidence triggered evolution
+    - The generator's reasoning
+    """
+    return f"""You are an independent skill quality reviewer. You have NOT seen the original
+version of this skill or why it was modified. Evaluate it purely on its merits.
+
+SKILL PURPOSE: {skill_name}
+EXPECTED BEHAVIOR: {skill_description}
+
+SKILL CONTENT TO REVIEW:
+{evolved_content[:8000]}
+
+EVALUATE:
+1. Does the skill clearly explain what to do? (clarity)
+2. Are the instructions specific and actionable? (specificity)
+3. Are there any obvious errors, contradictions, or missing steps? (correctness)
+4. Would an AI agent be able to follow these instructions? (executability)
+
+RESPOND IN JSON FORMAT:
+{{
+  "passed": true/false,
+  "confidence": 0.0-1.0,
+  "issues": ["issue1", "issue2"],
+  "reasoning": "brief explanation"
+}}
+
+Be strict. Only pass skills that are genuinely clear, correct, and actionable.
+A mediocre skill that might work sometimes should FAIL — evolution should produce
+clear improvements, not marginal changes."""
+
+
+def parse_verification_response(response: str) -> VerificationResult:
+    """Parse the verifier's JSON response."""
+    # Try parsing JSON from response
+    json_match = re.search(r"\{[^{}]*\"passed\"[^{}]*\}", response, re.DOTALL)
+    if json_match:
+        try:
+            data = json.loads(json_match.group(0))
+            return VerificationResult(
+                passed=bool(data.get("passed", False)),
+                confidence=float(data.get("confidence", 0.5)),
+                issues=tuple(data.get("issues", [])),
+                reasoning=str(data.get("reasoning", "")),
+            )
+        except (json.JSONDecodeError, TypeError, ValueError):
+            pass
+
+    # Fallback: keyword detection
+    lower = response.lower()
+    if any(kw in lower for kw in ("\"passed\": true", "passed: true", "approve", "looks good")):
+        return VerificationResult(passed=True, confidence=0.6, reasoning="keyword match")
+
+    if any(kw in lower for kw in ("\"passed\": false", "passed: false", "reject", "fail")):
+        return VerificationResult(passed=False, confidence=0.6, reasoning="keyword match")
+
+    # Default: reject if can't parse (conservative)
+    return VerificationResult(
+        passed=False,
+        confidence=0.3,
+        reasoning="Could not parse verification response",
+        issues=("Unparseable response",),
+    )

package/src/superlocalmemory/evolution/evolution_store.py

@@ -0,0 +1,302 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Evolution Store — SQLite persistence for skill evolution history.
+
+Stores evolution records, lineage DAG, and anti-loop state.
+Uses the same memory.db as the rest of SLM — no separate database.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from superlocalmemory.evolution.types import (
+    EvolutionCandidate,
+    EvolutionRecord,
+    EvolutionStatus,
+    EvolutionType,
+    TriggerType,
+)
+
+logger = logging.getLogger(__name__)
+
+_SCHEMA_DDL = """
+CREATE TABLE IF NOT EXISTS skill_evolution_log (
+    id TEXT PRIMARY KEY,
+    skill_name TEXT NOT NULL,
+    parent_skill_id TEXT,
+    evolution_type TEXT NOT NULL,
+    trigger_type TEXT NOT NULL,
+    generation INTEGER DEFAULT 0,
+    status TEXT DEFAULT 'candidate',
+    mutation_summary TEXT DEFAULT '',
+    evidence TEXT DEFAULT '[]',
+    original_content TEXT DEFAULT '',
+    evolved_content TEXT DEFAULT '',
+    content_diff TEXT DEFAULT '',
+    blind_verified INTEGER DEFAULT 0,
+    rejection_reason TEXT DEFAULT '',
+    created_at TEXT NOT NULL,
+    completed_at TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_evo_skill ON skill_evolution_log(skill_name);
+CREATE INDEX IF NOT EXISTS idx_evo_status ON skill_evolution_log(status);
+CREATE INDEX IF NOT EXISTS idx_evo_created ON skill_evolution_log(created_at);
+
+CREATE TABLE IF NOT EXISTS evolution_cycle_state (
+    key TEXT PRIMARY KEY,
+    value INTEGER DEFAULT 0,
+    updated_at TEXT
+);
+"""
+
+# Anti-loop budget
+MAX_EVOLUTIONS_PER_CYCLE = 3
+MAX_ATTEMPTS_PER_SKILL = 3
+MIN_FRESH_INVOCATIONS = 5
+
+
+class EvolutionStore:
+    """SQLite persistence for evolution history and anti-loop state."""
+
+    def __init__(self, db_path: str | Path):
+        self._db_path = str(db_path)
+        self._ensure_schema()
+        self._addressed_degradations: dict[str, set[str]] = {}
+
+    def _ensure_schema(self) -> None:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            conn.executescript(_SCHEMA_DDL)
+            conn.commit()
+        except sqlite3.OperationalError as exc:
+            logger.warning("Evolution schema creation failed: %s", exc)
+        finally:
+            conn.close()
+
+    def reset_cycle(self) -> None:
+        """Reset per-cycle counters. Call at start of each consolidation."""
+        now = datetime.now(timezone.utc).isoformat()
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            conn.execute(
+                "INSERT OR REPLACE INTO evolution_cycle_state (key, value, updated_at) "
+                "VALUES ('cycle_count', 0, ?)",
+                (now,),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def can_evolve(self) -> bool:
+        """Check if budget allows another evolution this cycle."""
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            row = conn.execute(
+                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+            ).fetchone()
+            count = row[0] if row else 0
+            return count < MAX_EVOLUTIONS_PER_CYCLE
+        finally:
+            conn.close()
+
+    def record_evolution_attempt(self) -> None:
+        """Increment cycle counter in DB."""
+        now = datetime.now(timezone.utc).isoformat()
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            row = conn.execute(
+                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+            ).fetchone()
+            current = row[0] if row else 0
+            conn.execute(
+                "INSERT OR REPLACE INTO evolution_cycle_state (key, value, updated_at) "
+                "VALUES ('cycle_count', ?, ?)",
+                (current + 1, now),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _get_cycle_count(self) -> int:
+        """Read current cycle count from DB."""
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            row = conn.execute(
+                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+            ).fetchone()
+            return row[0] if row else 0
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Anti-loop: addressed degradations (adopted from OpenSpace)
+    # ------------------------------------------------------------------
+
+    def is_addressed(self, skill_name: str, context_hash: str) -> bool:
+        return context_hash in self._addressed_degradations.get(skill_name, set())
+
+    def mark_addressed(self, skill_name: str, context_hash: str) -> None:
+        self._addressed_degradations.setdefault(skill_name, set()).add(context_hash)
+
+    def prune_recovered(self, active_degraded_skills: set[str]) -> None:
+        """Remove tracking for skills that recovered."""
+        recovered = [
+            k for k in self._addressed_degradations
+            if k not in active_degraded_skills
+        ]
+        for k in recovered:
+            del self._addressed_degradations[k]
+
+    # ------------------------------------------------------------------
+    # CRUD
+    # ------------------------------------------------------------------
+
+    def save_record(self, record: EvolutionRecord) -> None:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            conn.execute(
+                "INSERT OR REPLACE INTO skill_evolution_log "
+                "(id, skill_name, parent_skill_id, evolution_type, trigger_type, "
+                " generation, status, mutation_summary, evidence, "
+                " original_content, evolved_content, content_diff, "
+                " blind_verified, rejection_reason, created_at, completed_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    record.id,
+                    record.skill_name,
+                    record.parent_skill_id,
+                    record.evolution_type.value,
+                    record.trigger.value,
+                    record.generation,
+                    record.status.value,
+                    record.mutation_summary,
+                    json.dumps(list(record.evidence)),
+                    record.original_content,
+                    record.evolved_content,
+                    record.content_diff,
+                    1 if record.blind_verified else 0,
+                    record.rejection_reason,
+                    record.created_at,
+                    record.completed_at,
+                ),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def get_record(self, record_id: str) -> Optional[EvolutionRecord]:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        conn.row_factory = sqlite3.Row
+        try:
+            row = conn.execute(
+                "SELECT * FROM skill_evolution_log WHERE id = ?",
+                (record_id,),
+            ).fetchone()
+            if not row:
+                return None
+            return self._row_to_record(dict(row))
+        finally:
+            conn.close()
+
+    def get_skill_history(self, skill_name: str, limit: int = 20) -> list[EvolutionRecord]:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        conn.row_factory = sqlite3.Row
+        try:
+            rows = conn.execute(
+                "SELECT * FROM skill_evolution_log "
+                "WHERE skill_name = ? ORDER BY created_at DESC LIMIT ?",
+                (skill_name, limit),
+            ).fetchall()
+            return [self._row_to_record(dict(r)) for r in rows]
+        finally:
+            conn.close()
+
+    def get_recent(self, limit: int = 10) -> list[EvolutionRecord]:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        conn.row_factory = sqlite3.Row
+        try:
+            rows = conn.execute(
+                "SELECT * FROM skill_evolution_log "
+                "ORDER BY created_at DESC LIMIT ?",
+                (limit,),
+            ).fetchall()
+            return [self._row_to_record(dict(r)) for r in rows]
+        finally:
+            conn.close()
+
+    def count_attempts(self, skill_name: str) -> int:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            row = conn.execute(
+                "SELECT COUNT(*) FROM skill_evolution_log "
+                "WHERE skill_name = ? AND status NOT IN ('promoted')",
+                (skill_name,),
+            ).fetchone()
+            return row[0] if row else 0
+        finally:
+            conn.close()
+
+    def has_exceeded_attempts(self, skill_name: str) -> bool:
+        return self.count_attempts(skill_name) >= MAX_ATTEMPTS_PER_SKILL
+
+    def get_stats(self) -> dict:
+        conn = sqlite3.connect(self._db_path, timeout=10)
+        try:
+            total = conn.execute(
+                "SELECT COUNT(*) FROM skill_evolution_log",
+            ).fetchone()[0]
+            by_status = {}
+            for row in conn.execute(
+                "SELECT status, COUNT(*) FROM skill_evolution_log GROUP BY status",
+            ).fetchall():
+                by_status[row[0]] = row[1]
+            by_type = {}
+            for row in conn.execute(
+                "SELECT evolution_type, COUNT(*) FROM skill_evolution_log GROUP BY evolution_type",
+            ).fetchall():
+                by_type[row[0]] = row[1]
+            return {
+                "total": total,
+                "by_status": by_status,
+                "by_type": by_type,
+                "cycle_budget_remaining": MAX_EVOLUTIONS_PER_CYCLE - self._get_cycle_count(),
+            }
+        finally:
+            conn.close()
+
+    def _row_to_record(self, row: dict) -> EvolutionRecord:
+        evidence_raw = row.get("evidence", "[]")
+        try:
+            evidence = tuple(json.loads(evidence_raw))
+        except (json.JSONDecodeError, TypeError):
+            evidence = ()
+
+        return EvolutionRecord(
+            id=row["id"],
+            skill_name=row["skill_name"],
+            parent_skill_id=row.get("parent_skill_id"),
+            evolution_type=EvolutionType(row["evolution_type"]),
+            trigger=TriggerType(row["trigger_type"]),
+            generation=row.get("generation", 0),
+            status=EvolutionStatus(row.get("status", "candidate")),
+            mutation_summary=row.get("mutation_summary", ""),
+            evidence=evidence,
+            original_content=row.get("original_content", ""),
+            evolved_content=row.get("evolved_content", ""),
+            content_diff=row.get("content_diff", ""),
+            blind_verified=bool(row.get("blind_verified", 0)),
+            rejection_reason=row.get("rejection_reason", ""),
+            created_at=row.get("created_at", ""),
+            completed_at=row.get("completed_at"),
+        )