superlocalmemory 3.7.7 → 3.8.0

package/src/superlocalmemory/evolution/evolution_store.py

@@ -32,6 +32,7 @@ logger = logging.getLogger(__name__)
 _SCHEMA_DDL = """
 CREATE TABLE IF NOT EXISTS skill_evolution_log (
     id TEXT PRIMARY KEY,
+    profile_id TEXT NOT NULL DEFAULT 'default',
     skill_name TEXT NOT NULL,
     parent_skill_id TEXT,
     evolution_type TEXT NOT NULL,
@@ -49,14 +50,16 @@ CREATE TABLE IF NOT EXISTS skill_evolution_log (
     completed_at TEXT
 );
 
-CREATE INDEX IF NOT EXISTS idx_evo_skill ON skill_evolution_log(skill_name);
-CREATE INDEX IF NOT EXISTS idx_evo_status ON skill_evolution_log(status);
-CREATE INDEX IF NOT EXISTS idx_evo_created ON skill_evolution_log(created_at);
+CREATE INDEX IF NOT EXISTS idx_evo_skill ON skill_evolution_log(profile_id, skill_name);
+CREATE INDEX IF NOT EXISTS idx_evo_status ON skill_evolution_log(profile_id, status);
+CREATE INDEX IF NOT EXISTS idx_evo_created ON skill_evolution_log(profile_id, created_at);
 
 CREATE TABLE IF NOT EXISTS evolution_cycle_state (
-    key TEXT PRIMARY KEY,
+    profile_id TEXT NOT NULL DEFAULT 'default',
+    key TEXT NOT NULL,
     value INTEGER DEFAULT 0,
-    updated_at TEXT
+    updated_at TEXT,
+    PRIMARY KEY (profile_id, key)
 );
 """
 
@@ -77,6 +80,11 @@ class EvolutionStore:
     def _ensure_schema(self) -> None:
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
+            # Migrate an existing pre-isolation DB BEFORE running the schema DDL:
+            # the DDL creates indexes on profile_id, which would fail on a legacy
+            # table that still lacks the column. On a fresh DB the migration is a
+            # no-op (tables absent) and the DDL creates everything correctly.
+            self._migrate_profile_isolation(conn)
             conn.executescript(_SCHEMA_DDL)
             conn.commit()
         except sqlite3.OperationalError as exc:
@@ -84,56 +92,113 @@ class EvolutionStore:
         finally:
             conn.close()
 
-    def reset_cycle(self) -> None:
-        """Reset per-cycle counters. Call at start of each consolidation."""
+    def _migrate_profile_isolation(self, conn: sqlite3.Connection) -> None:
+        """Self-migrate pre-profile-isolation DBs to add per-profile scoping.
+
+        These tables are store-owned (created lazily here, not by the schema
+        migration runner which fires before this store exists), so the store
+        owns their upgrade too. Idempotent: only alters when the column/PK is
+        missing. Existing rows backfill to 'default' — the historical profile.
+        """
+        # skill_evolution_log: additive column (safe, preserves rows).
+        cols = {r[1] for r in conn.execute(
+            "PRAGMA table_info(skill_evolution_log)").fetchall()}
+        if cols and "profile_id" not in cols:
+            conn.execute(
+                "ALTER TABLE skill_evolution_log "
+                "ADD COLUMN profile_id TEXT NOT NULL DEFAULT 'default'"
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_evo_skill "
+                "ON skill_evolution_log(profile_id, skill_name)"
+            )
+
+        # evolution_cycle_state: PK changes from (key) to (profile_id, key).
+        # SQLite cannot alter a PK in place → rebuild + copy, backfilling
+        # existing counters to the 'default' profile.
+        cyc_cols = {r[1] for r in conn.execute(
+            "PRAGMA table_info(evolution_cycle_state)").fetchall()}
+        if cyc_cols and "profile_id" not in cyc_cols:
+            conn.execute(
+                "ALTER TABLE evolution_cycle_state RENAME TO _evo_cycle_old"
+            )
+            conn.execute(
+                "CREATE TABLE evolution_cycle_state ("
+                " profile_id TEXT NOT NULL DEFAULT 'default',"
+                " key TEXT NOT NULL,"
+                " value INTEGER DEFAULT 0,"
+                " updated_at TEXT,"
+                " PRIMARY KEY (profile_id, key))"
+            )
+            conn.execute(
+                "INSERT INTO evolution_cycle_state "
+                "(profile_id, key, value, updated_at) "
+                "SELECT 'default', key, value, updated_at FROM _evo_cycle_old"
+            )
+            conn.execute("DROP TABLE _evo_cycle_old")
+
+    def reset_cycle(self, profile_id: str) -> None:
+        """Reset per-cycle counters for one profile. Called at cycle start.
+
+        The evolve budget is per-profile: one profile exhausting its budget
+        must never block another profile from evolving.
+        """
         now = datetime.now(timezone.utc).isoformat()
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             conn.execute(
-                "INSERT OR REPLACE INTO evolution_cycle_state (key, value, updated_at) "
-                "VALUES ('cycle_count', 0, ?)",
-                (now,),
+                "INSERT OR REPLACE INTO evolution_cycle_state "
+                "(profile_id, key, value, updated_at) "
+                "VALUES (?, 'cycle_count', 0, ?)",
+                (profile_id, now),
             )
             conn.commit()
         finally:
             conn.close()
 
-    def can_evolve(self) -> bool:
-        """Check if budget allows another evolution this cycle."""
+    def can_evolve(self, profile_id: str) -> bool:
+        """Check if this profile's budget allows another evolution this cycle."""
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             row = conn.execute(
-                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+                "SELECT value FROM evolution_cycle_state "
+                "WHERE profile_id = ? AND key = 'cycle_count'",
+                (profile_id,),
             ).fetchone()
             count = row[0] if row else 0
             return count < MAX_EVOLUTIONS_PER_CYCLE
         finally:
             conn.close()
 
-    def record_evolution_attempt(self) -> None:
-        """Increment cycle counter in DB."""
+    def record_evolution_attempt(self, profile_id: str) -> None:
+        """Increment this profile's cycle counter in DB."""
         now = datetime.now(timezone.utc).isoformat()
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             row = conn.execute(
-                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+                "SELECT value FROM evolution_cycle_state "
+                "WHERE profile_id = ? AND key = 'cycle_count'",
+                (profile_id,),
             ).fetchone()
             current = row[0] if row else 0
             conn.execute(
-                "INSERT OR REPLACE INTO evolution_cycle_state (key, value, updated_at) "
-                "VALUES ('cycle_count', ?, ?)",
-                (current + 1, now),
+                "INSERT OR REPLACE INTO evolution_cycle_state "
+                "(profile_id, key, value, updated_at) "
+                "VALUES (?, 'cycle_count', ?, ?)",
+                (profile_id, current + 1, now),
             )
             conn.commit()
         finally:
             conn.close()
 
-    def _get_cycle_count(self) -> int:
-        """Read current cycle count from DB."""
+    def _get_cycle_count(self, profile_id: str) -> int:
+        """Read this profile's current cycle count from DB."""
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             row = conn.execute(
-                "SELECT value FROM evolution_cycle_state WHERE key = 'cycle_count'",
+                "SELECT value FROM evolution_cycle_state "
+                "WHERE profile_id = ? AND key = 'cycle_count'",
+                (profile_id,),
             ).fetchone()
             return row[0] if row else 0
         finally:
@@ -162,18 +227,19 @@ class EvolutionStore:
     # CRUD
     # ------------------------------------------------------------------
 
-    def save_record(self, record: EvolutionRecord) -> None:
+    def save_record(self, record: EvolutionRecord, profile_id: str) -> None:
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             conn.execute(
                 "INSERT OR REPLACE INTO skill_evolution_log "
-                "(id, skill_name, parent_skill_id, evolution_type, trigger_type, "
-                " generation, status, mutation_summary, evidence, "
+                "(id, profile_id, skill_name, parent_skill_id, evolution_type, "
+                " trigger_type, generation, status, mutation_summary, evidence, "
                 " original_content, evolved_content, content_diff, "
                 " blind_verified, rejection_reason, created_at, completed_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                 (
                     record.id,
+                    profile_id,
                     record.skill_name,
                     record.parent_skill_id,
                     record.evolution_type.value,
@@ -195,13 +261,14 @@ class EvolutionStore:
         finally:
             conn.close()
 
-    def get_record(self, record_id: str) -> Optional[EvolutionRecord]:
+    def get_record(self, record_id: str, profile_id: str) -> Optional[EvolutionRecord]:
         conn = sqlite3.connect(self._db_path, timeout=10)
         conn.row_factory = sqlite3.Row
         try:
             row = conn.execute(
-                "SELECT * FROM skill_evolution_log WHERE id = ?",
-                (record_id,),
+                "SELECT * FROM skill_evolution_log "
+                "WHERE id = ? AND profile_id = ?",
+                (record_id, profile_id),
             ).fetchone()
             if not row:
                 return None
@@ -209,68 +276,78 @@ class EvolutionStore:
         finally:
             conn.close()
 
-    def get_skill_history(self, skill_name: str, limit: int = 20) -> list[EvolutionRecord]:
+    def get_skill_history(
+        self, skill_name: str, profile_id: str, limit: int = 20,
+    ) -> list[EvolutionRecord]:
         conn = sqlite3.connect(self._db_path, timeout=10)
         conn.row_factory = sqlite3.Row
         try:
             rows = conn.execute(
                 "SELECT * FROM skill_evolution_log "
-                "WHERE skill_name = ? ORDER BY created_at DESC LIMIT ?",
-                (skill_name, limit),
+                "WHERE skill_name = ? AND profile_id = ? "
+                "ORDER BY created_at DESC LIMIT ?",
+                (skill_name, profile_id, limit),
             ).fetchall()
             return [self._row_to_record(dict(r)) for r in rows]
         finally:
             conn.close()
 
-    def get_recent(self, limit: int = 10) -> list[EvolutionRecord]:
+    def get_recent(self, profile_id: str, limit: int = 10) -> list[EvolutionRecord]:
         conn = sqlite3.connect(self._db_path, timeout=10)
         conn.row_factory = sqlite3.Row
         try:
             rows = conn.execute(
                 "SELECT * FROM skill_evolution_log "
-                "ORDER BY created_at DESC LIMIT ?",
-                (limit,),
+                "WHERE profile_id = ? ORDER BY created_at DESC LIMIT ?",
+                (profile_id, limit),
             ).fetchall()
             return [self._row_to_record(dict(r)) for r in rows]
         finally:
             conn.close()
 
-    def count_attempts(self, skill_name: str) -> int:
+    def count_attempts(self, skill_name: str, profile_id: str) -> int:
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             row = conn.execute(
                 "SELECT COUNT(*) FROM skill_evolution_log "
-                "WHERE skill_name = ? AND status NOT IN ('promoted')",
-                (skill_name,),
+                "WHERE skill_name = ? AND profile_id = ? "
+                "AND status NOT IN ('promoted')",
+                (skill_name, profile_id),
             ).fetchone()
             return row[0] if row else 0
         finally:
             conn.close()
 
-    def has_exceeded_attempts(self, skill_name: str) -> bool:
-        return self.count_attempts(skill_name) >= MAX_ATTEMPTS_PER_SKILL
+    def has_exceeded_attempts(self, skill_name: str, profile_id: str) -> bool:
+        return self.count_attempts(skill_name, profile_id) >= MAX_ATTEMPTS_PER_SKILL
 
-    def get_stats(self) -> dict:
+    def get_stats(self, profile_id: str) -> dict:
         conn = sqlite3.connect(self._db_path, timeout=10)
         try:
             total = conn.execute(
-                "SELECT COUNT(*) FROM skill_evolution_log",
+                "SELECT COUNT(*) FROM skill_evolution_log WHERE profile_id = ?",
+                (profile_id,),
             ).fetchone()[0]
             by_status = {}
             for row in conn.execute(
-                "SELECT status, COUNT(*) FROM skill_evolution_log GROUP BY status",
+                "SELECT status, COUNT(*) FROM skill_evolution_log "
+                "WHERE profile_id = ? GROUP BY status",
+                (profile_id,),
             ).fetchall():
                 by_status[row[0]] = row[1]
             by_type = {}
             for row in conn.execute(
-                "SELECT evolution_type, COUNT(*) FROM skill_evolution_log GROUP BY evolution_type",
+                "SELECT evolution_type, COUNT(*) FROM skill_evolution_log "
+                "WHERE profile_id = ? GROUP BY evolution_type",
+                (profile_id,),
             ).fetchall():
                 by_type[row[0]] = row[1]
             return {
                 "total": total,
                 "by_status": by_status,
                 "by_type": by_type,
-                "cycle_budget_remaining": MAX_EVOLUTIONS_PER_CYCLE - self._get_cycle_count(),
+                "cycle_budget_remaining":
+                    MAX_EVOLUTIONS_PER_CYCLE - self._get_cycle_count(profile_id),
             }
         finally:
             conn.close()

package/src/superlocalmemory/evolution/llm_dispatch.py

@@ -162,7 +162,18 @@ ALLOWED_LLM_MODELS: frozenset[str] = frozenset({
 
 FORBIDDEN_MODEL_SUBSTRINGS: tuple[str, ...] = ("opus", "gpt-4-turbo")
 
-MAX_TOKENS_CAP: int = 500
+# Hard ceiling on ``max_tokens`` for any single evolution LLM call
+# (LLD-11). This is a CEILING, not a per-call default: each caller passes
+# its own budget (confirm=100, blind-verify=500, mutation=4000). The value
+# MUST stay >= the largest legitimate caller request — otherwise that call
+# trips the guard in ``_dispatch_llm`` and silently fails-closed. The
+# mutation step (``SkillEvolver._generate_mutation``) generates a full
+# SKILL.md and requests 4000; a stale 500 here made mutation return "" on
+# every cycle, killing evolution invisibly. 8000 accommodates 4000 with
+# retry/large-skill headroom. Do NOT lower below 4096 without first
+# shrinking that caller's request. Per-call spend is still bounded by
+# EvolutionBudget (10 LLM calls/cycle x 3 cycles/day).
+MAX_TOKENS_CAP: int = 8000
 
 
 # ---------------------------------------------------------------------------

package/src/superlocalmemory/evolution/model_selection.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.7.9 — evolution model selection
+
+"""Per-step LLM model selection for the skill-evolution pipeline.
+
+v3.7.9: skill evolution defaults to the *lowest-cost* capable model for
+whatever backend is active, and lets users override each step
+(``evolution.mutation_model`` / ``verify_model`` / ``confirm_model``).
+
+Two invariants this module enforces:
+
+1. **Lowest cost by default.** Claude backend → Haiku; Ollama → a local
+   (free) model. No step silently defaults to a premium model.
+2. **Generator != verifier.** Blind verification must run on a *different*
+   model from the generator, otherwise the generator effectively grades
+   its own homework. When an independent verifier isn't available for
+   free (Claude backend, no local Ollama), we reuse the cheapest model
+   and set ``independent=False`` so the caller can log the reduced-safety
+   condition — we never escalate the verifier to a premium model just to
+   force independence (that would defeat the cost goal).
+
+This module is pure: it takes ``ollama_available`` as an argument instead
+of probing, so it is fully unit-testable without network access.
+
+Author: Varun Pratap Bhardwaj / Qualixar
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+# Allow-listed model ids (must stay a subset of
+# ``llm_dispatch.ALLOWED_LLM_MODELS``).
+CHEAPEST_CLAUDE = "claude-haiku-4-5"
+QUALITY_CLAUDE = "claude-sonnet-4-6"
+CHEAPEST_OLLAMA = "ollama:llama3"
+ALT_OLLAMA = "ollama:qwen2.5"
+
+# Short-name → allow-listed model id. Single source of truth for aliasing;
+# ``skill_evolver`` re-exports these for backward compatibility.
+_MODEL_ALIASES: dict[str, str] = {
+    "haiku": CHEAPEST_CLAUDE,
+    "sonnet": QUALITY_CLAUDE,
+    "ollama": CHEAPEST_OLLAMA,
+    "ollama:llama3": CHEAPEST_OLLAMA,
+    "ollama:qwen2.5": ALT_OLLAMA,
+    CHEAPEST_CLAUDE: CHEAPEST_CLAUDE,
+    QUALITY_CLAUDE: QUALITY_CLAUDE,
+}
+
+# Models this subsystem is allowed to dispatch. Anything resolving outside
+# this set falls back to the cheapest safe default rather than dispatching
+# an unvetted (or forbidden) model.
+_ALLOWED: frozenset[str] = frozenset(_MODEL_ALIASES.values())
+
+
+def _resolve_model_alias(alias: str) -> str:
+    """Translate a short alias (``haiku``/``sonnet``) to an allow-listed id.
+
+    Unknown aliases fall back to the cheapest Claude model — the same
+    fail-safe the evolver has always used.
+    """
+    return _MODEL_ALIASES.get(alias, CHEAPEST_CLAUDE)
+
+
+@dataclass(frozen=True)
+class ResolvedModels:
+    """Concrete allow-listed model id for each pipeline step."""
+
+    mutation: str
+    verify: str
+    confirm: str
+    independent: bool  # True when verify != mutation (blind-verify is sound)
+
+
+def _cheapest_for_backend(backend: str) -> str:
+    """Lowest-cost allow-listed model for a detected backend."""
+    if backend == "ollama":
+        return CHEAPEST_OLLAMA
+    # claude CLI, anthropic API, and any non-Ollama backend evolution can
+    # actually dispatch route through the cheapest Claude model.
+    return CHEAPEST_CLAUDE
+
+
+def _resolve_field(value: str, default: str) -> str:
+    """Resolve a config field: empty → default; else alias→id, validated."""
+    if not value:
+        return default
+    resolved = _resolve_model_alias(value)
+    if resolved not in _ALLOWED:  # pragma: no cover — defensive
+        logger.warning(
+            "evolution: configured model %r is not allow-listed; "
+            "falling back to %s", value, default,
+        )
+        return default
+    return resolved
+
+
+def _independent_verifier(
+    mutation: str, *, ollama_available: bool,
+) -> str:
+    """Pick a cheap verifier that differs from the generator when possible.
+
+    - Ollama generator → the *other* local model (both free, distinct).
+    - Claude generator + local Ollama up → free local verifier.
+    - Claude generator, no Ollama → the cheapest *different* Claude tier if
+      the generator was a premium model; otherwise reuse the cheapest model
+      (caller flags reduced independence — we don't pay for a premium
+      verifier just to force distinctness).
+    """
+    if mutation.startswith("ollama:"):
+        return ALT_OLLAMA if mutation != ALT_OLLAMA else CHEAPEST_OLLAMA
+    if ollama_available:
+        return CHEAPEST_OLLAMA
+    if mutation != CHEAPEST_CLAUDE:
+        return CHEAPEST_CLAUDE
+    return CHEAPEST_CLAUDE
+
+
+def resolve_evolution_models(
+    config: object, backend: str, *, ollama_available: bool,
+) -> ResolvedModels:
+    """Resolve mutation/verify/confirm models for a backend.
+
+    ``config`` is an ``EvolutionConfig`` (duck-typed: reads
+    ``mutation_model`` / ``verify_model`` / ``confirm_model`` attributes,
+    each an alias, allow-listed id, or "" for auto).
+    """
+    cheapest = _cheapest_for_backend(backend)
+
+    mutation = _resolve_field(getattr(config, "mutation_model", ""), cheapest)
+    confirm = _resolve_field(getattr(config, "confirm_model", ""), cheapest)
+
+    verify_cfg = getattr(config, "verify_model", "")
+    if verify_cfg:
+        verify = _resolve_field(verify_cfg, cheapest)
+    else:
+        verify = _independent_verifier(
+            mutation, ollama_available=ollama_available,
+        )
+
+    independent = verify != mutation
+    if not independent:
+        logger.warning(
+            "evolution: blind-verify model == generator model (%s) — "
+            "blind-verification independence is reduced. Set "
+            "evolution.verify_model to a different model, or run Ollama "
+            "locally, for an independent verifier.",
+            mutation,
+        )
+    return ResolvedModels(
+        mutation=mutation,
+        verify=verify,
+        confirm=confirm,
+        independent=independent,
+    )

package/src/superlocalmemory/evolution/mutation_generator.py

@@ -89,6 +89,17 @@ def parse_mutation_output(output: str) -> Optional[str]:
     return None
 
 
+# An evolved SKILL.md is auto-loaded into future sessions, so a mutation that
+# hallucinated (or was prompt-injected into producing) code-execution or
+# secret-exfiltration instructions must never be persisted. Structure checks
+# alone are insufficient — reject dangerous content outright.
+_SKILL_DENY_PATTERNS: tuple[str, ...] = (
+    "os.environ", "subprocess", "exec(", "eval(", "__import__", "import os",
+    "import subprocess", "pickle.loads", "curl ", "wget ", "rm -rf",
+    "ANTHROPIC_API_KEY", "OPENAI_API_KEY", "AWS_SECRET", "/.ssh/", ".install_token",
+)
+
+
 def validate_skill_content(content: str) -> Optional[str]:
     """Validate evolved skill content. Returns error message or None if valid."""
     if not content or len(content) < 50:
@@ -97,6 +108,11 @@ def validate_skill_content(content: str) -> Optional[str]:
         return "Missing YAML frontmatter (no --- found)"
     if content.count("---") >= 2 and "name:" not in content.split("---")[1]:
         return "Missing 'name:' in frontmatter"
+    lowered = content.lower()
+    for pat in _SKILL_DENY_PATTERNS:
+        if pat.lower() in lowered:
+            return (f"Rejected: evolved skill contains a disallowed pattern "
+                    f"({pat!r}) — skills must not execute code or access secrets")
     return None