superlocalmemory 3.2.3 → 3.3.0

package/src/superlocalmemory/storage/database.py

@@ -962,3 +962,209 @@ class DatabaseManager:
             "DELETE FROM core_memory_blocks WHERE profile_id = ?",
             (profile_id,),
         )
+
+    # ------------------------------------------------------------------
+    # Phase A: Fact Retention CRUD (Forgetting Brain)
+    # ------------------------------------------------------------------
+
+    def get_retention(self, fact_id: str, profile_id: str) -> dict | None:
+        """Get retention data for a single fact.
+
+        Returns dict with column names as keys, or None if not found.
+        All SQL parameterized (HR-05).
+        """
+        rows = self.execute(
+            "SELECT fact_id, retention_score, memory_strength, access_count, "
+            "       last_accessed_at, lifecycle_zone, last_computed_at "
+            "FROM fact_retention WHERE fact_id = ? AND profile_id = ?",
+            (fact_id, profile_id),
+        )
+        return dict(rows[0]) if rows else None
+
+    def batch_get_retention(
+        self, fact_ids: list[str], profile_id: str,
+    ) -> list[dict]:
+        """Get retention data for a batch of facts.
+
+        Uses dynamic ? placeholders for IN clause (never string concat).
+        Missing fact_ids are simply absent from results.
+        All SQL parameterized (HR-05).
+        """
+        if not fact_ids:
+            return []
+        placeholders = ",".join("?" for _ in fact_ids)
+        rows = self.execute(
+            f"SELECT fact_id, retention_score, lifecycle_zone "
+            f"FROM fact_retention "
+            f"WHERE fact_id IN ({placeholders}) AND profile_id = ?",
+            (*fact_ids, profile_id),
+        )
+        return [dict(r) for r in rows]
+
+    def upsert_retention(
+        self,
+        fact_id: str,
+        profile_id: str,
+        retention_score: float,
+        memory_strength: float,
+        access_count: int,
+        last_accessed_at: str,
+        lifecycle_zone: str,
+    ) -> None:
+        """UPSERT retention data for a fact.
+
+        Retries 3x on SQLITE_BUSY (handled by execute()).
+        All SQL parameterized (HR-05).
+        """
+        self.execute(
+            "INSERT INTO fact_retention "
+            "(fact_id, profile_id, retention_score, memory_strength, "
+            " access_count, last_accessed_at, lifecycle_zone, last_computed_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now')) "
+            "ON CONFLICT(fact_id) DO UPDATE SET "
+            "  retention_score = excluded.retention_score, "
+            "  memory_strength = excluded.memory_strength, "
+            "  access_count = excluded.access_count, "
+            "  lifecycle_zone = excluded.lifecycle_zone, "
+            "  last_computed_at = excluded.last_computed_at",
+            (fact_id, profile_id, retention_score, memory_strength,
+             access_count, last_accessed_at, lifecycle_zone),
+        )
+
+    def batch_upsert_retention(
+        self, facts: list[dict], profile_id: str,
+    ) -> int:
+        """Batch UPSERT retention data. Wraps in transaction for atomicity.
+
+        Each dict must contain: fact_id, retention, strength,
+        access_count, last_accessed_at, zone.
+
+        Returns count of successfully upserted rows.
+        """
+        count = 0
+        with self.transaction():
+            for f in facts:
+                self.upsert_retention(
+                    fact_id=f["fact_id"],
+                    profile_id=profile_id,
+                    retention_score=f["retention"],
+                    memory_strength=f["strength"],
+                    access_count=f["access_count"],
+                    last_accessed_at=f["last_accessed_at"],
+                    lifecycle_zone=f["zone"],
+                )
+                count += 1
+        return count
+
+    def get_facts_needing_decay(self, profile_id: str) -> list[dict]:
+        """Get facts that need decay computation (excludes core memory).
+
+        Core memory facts are immune to forgetting (HR-01).
+        All SQL parameterized (HR-05).
+        """
+        rows = self.execute(
+            "SELECT f.fact_id, f.created_at, f.profile_id "
+            "FROM atomic_facts f "
+            "LEFT JOIN fact_retention r ON f.fact_id = r.fact_id "
+            "WHERE f.profile_id = ? "
+            "AND f.fact_id NOT IN ("
+            "  SELECT json_each.value "
+            "  FROM core_memory_blocks, json_each(core_memory_blocks.source_fact_ids) "
+            "  WHERE core_memory_blocks.profile_id = ?"
+            ")",
+            (profile_id, profile_id),
+        )
+        return [dict(r) for r in rows]
+
+    def soft_delete_fact(self, fact_id: str, profile_id: str) -> None:
+        """Soft-delete a forgotten fact.
+
+        Sets fact_retention.lifecycle_zone to 'forgotten' and
+        atomic_facts.lifecycle to 'archived' (valid enum value).
+        Never physically deletes (HR-04).
+
+        Idempotent: if fact not found, logs warning and returns.
+        """
+        # Check existence first (idempotent)
+        rows = self.execute(
+            "SELECT fact_id FROM fact_retention WHERE fact_id = ? AND profile_id = ?",
+            (fact_id, profile_id),
+        )
+        if not rows:
+            logger.warning(
+                "soft_delete_fact: fact_id=%s not found in fact_retention, skipping",
+                fact_id,
+            )
+            return
+
+        # Update fact_retention
+        self.execute(
+            "UPDATE fact_retention SET lifecycle_zone = 'forgotten', "
+            "  retention_score = 0.0 "
+            "WHERE fact_id = ? AND profile_id = ?",
+            (fact_id, profile_id),
+        )
+
+        # Mark in atomic_facts as archived (valid enum value per A-CRIT-01)
+        self.execute(
+            "UPDATE atomic_facts SET lifecycle = 'archived' "
+            "WHERE fact_id = ? AND profile_id = ?",
+            (fact_id, profile_id),
+        )
+
+    # ------------------------------------------------------------------
+    # Phase E: CCQ Consolidated Blocks & Audit CRUD
+    # ------------------------------------------------------------------
+
+    def store_ccq_block(
+        self,
+        block_id: str,
+        profile_id: str,
+        content: str,
+        source_fact_ids: str,
+        gist_embedding_rowid: int | None,
+        char_count: int,
+        cluster_id: str,
+    ) -> None:
+        """Store a CCQ consolidated block. Parameterized SQL only."""
+        self.execute(
+            "INSERT INTO ccq_consolidated_blocks "
+            "(block_id, profile_id, content, source_fact_ids, "
+            " gist_embedding_rowid, char_count, compiled_by, cluster_id, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, 'ccq', ?, datetime('now'))",
+            (block_id, profile_id, content, source_fact_ids,
+             gist_embedding_rowid, char_count, cluster_id),
+        )
+
+    def get_ccq_blocks(self, profile_id: str) -> list[dict]:
+        """Get all CCQ consolidated blocks for a profile."""
+        rows = self.execute(
+            "SELECT * FROM ccq_consolidated_blocks "
+            "WHERE profile_id = ? ORDER BY created_at DESC",
+            (profile_id,),
+        )
+        return [dict(r) for r in rows]
+
+    def store_ccq_audit(self, entry: dict) -> None:
+        """Store a CCQ audit log entry. Parameterized SQL only."""
+        self.execute(
+            "INSERT INTO ccq_audit_log "
+            "(audit_id, profile_id, cluster_id, block_id, fact_ids, fact_count, "
+            " gist_text, extraction_mode, bytes_before, bytes_after, "
+            " compression_ratio, shared_entities, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))",
+            (entry["audit_id"], entry["profile_id"], entry["cluster_id"],
+             entry["block_id"], entry["fact_ids"], entry["fact_count"],
+             entry["gist_text"], entry["extraction_mode"],
+             entry["bytes_before"], entry["bytes_after"],
+             entry["compression_ratio"], entry["shared_entities"]),
+        )
+
+    def get_ccq_audit(self, profile_id: str, limit: int = 50) -> list[dict]:
+        """Get CCQ audit log entries for a profile."""
+        rows = self.execute(
+            "SELECT * FROM ccq_audit_log "
+            "WHERE profile_id = ? ORDER BY created_at DESC LIMIT ?",
+            (profile_id, limit),
+        )
+        return [dict(r) for r in rows]

package/src/superlocalmemory/storage/embedding_migrator.py

@@ -0,0 +1,178 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Embedding migration on mode/model switch.
+
+When a user switches modes (e.g., Mode B Ollama -> Mode A sentence-transformers),
+the embeddings live in different vector spaces. This module detects the mismatch
+and flags facts for progressive re-embedding.
+
+Key table: ``embedding_metadata.model_name`` stores the model used for each fact.
+A config-level field in ``config.json`` stores the current model signature.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from superlocalmemory.core.config import SLMConfig
+
+logger = logging.getLogger(__name__)
+
+# Sentinel stored in config.json when no model has been set yet.
+_NO_MODEL = ""
+
+# Batch size for progressive re-embedding.
+_REINDEX_BATCH_SIZE = 50
+
+
+def _model_signature(config: SLMConfig) -> str:
+    """Derive a deterministic signature from the active embedding config.
+
+    The signature combines provider + model_name + dimension so that
+    any change in embedding source is detected.
+    """
+    emb = config.embedding
+    return f"{emb.provider}::{emb.model_name}::{emb.dimension}"
+
+
+def _read_stored_signature(config_dir: Path) -> str:
+    """Read the last-used embedding model signature from config.json."""
+    config_path = config_dir / "config.json"
+    if not config_path.exists():
+        return _NO_MODEL
+    try:
+        data = json.loads(config_path.read_text())
+        return data.get("embedding_signature", _NO_MODEL)
+    except (json.JSONDecodeError, OSError):
+        return _NO_MODEL
+
+
+def _write_stored_signature(config_dir: Path, signature: str) -> None:
+    """Persist the current embedding model signature to config.json."""
+    config_path = config_dir / "config.json"
+    data: dict[str, Any] = {}
+    if config_path.exists():
+        try:
+            data = json.loads(config_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    data["embedding_signature"] = signature
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    config_path.write_text(json.dumps(data, indent=2))
+
+
+def check_embedding_migration(config: SLMConfig) -> bool:
+    """Check if embedding model changed since last run.
+
+    Returns True if re-indexing is needed (model signature differs).
+    Returns False if signatures match or this is the first run.
+    """
+    current_sig = _model_signature(config)
+    stored_sig = _read_stored_signature(config.base_dir)
+
+    if stored_sig == _NO_MODEL:
+        # First run — store signature, no migration needed.
+        _write_stored_signature(config.base_dir, current_sig)
+        logger.info("Embedding signature initialized: %s", current_sig)
+        return False
+
+    if stored_sig == current_sig:
+        return False
+
+    logger.warning(
+        "Embedding model changed: %s -> %s. Re-indexing required.",
+        stored_sig, current_sig,
+    )
+    return True
+
+
+def run_embedding_migration(
+    config: SLMConfig,
+    db: Any,
+    embedder: Any,
+) -> int:
+    """Re-embed all facts with the current model. Returns count re-embedded.
+
+    Processes facts in batches to avoid memory spikes. Updates the
+    embedding_metadata table and vector store for each fact.
+
+    This is idempotent — can be interrupted and resumed safely.
+    """
+    if embedder is None:
+        logger.warning("No embedder available. Skipping re-indexing.")
+        return 0
+
+    current_sig = _model_signature(config)
+    profile_id = config.active_profile
+
+    # Get all fact IDs that need re-embedding (all facts for the profile).
+    rows = db.execute(
+        "SELECT fact_id, content FROM atomic_facts "
+        "WHERE profile_id = ? ORDER BY created_at",
+        (profile_id,),
+    )
+    facts = [(dict(r)["fact_id"], dict(r)["content"]) for r in rows]
+    total = len(facts)
+
+    if total == 0:
+        _write_stored_signature(config.base_dir, current_sig)
+        return 0
+
+    logger.info(
+        "Re-embedding %d facts with model %s (batch_size=%d)",
+        total, current_sig, _REINDEX_BATCH_SIZE,
+    )
+
+    reindexed = 0
+    for i in range(0, total, _REINDEX_BATCH_SIZE):
+        batch = facts[i : i + _REINDEX_BATCH_SIZE]
+        texts = [content for _, content in batch]
+        fact_ids = [fid for fid, _ in batch]
+
+        try:
+            vectors = embedder.embed_batch(texts)
+        except Exception as exc:
+            logger.error(
+                "Re-embedding batch %d-%d failed: %s. Stopping migration.",
+                i, i + len(batch), exc,
+            )
+            break
+
+        for j, (fid, vec) in enumerate(zip(fact_ids, vectors)):
+            if vec is None:
+                continue
+            # Update embedding in the database (embedding column on atomic_facts).
+            try:
+                embedding_json = json.dumps(vec)
+                db.execute(
+                    "UPDATE atomic_facts SET embedding = ? WHERE fact_id = ?",
+                    (embedding_json, fid),
+                )
+                # Update embedding_metadata with new model name.
+                db.execute(
+                    "UPDATE embedding_metadata SET model_name = ? "
+                    "WHERE fact_id = ?",
+                    (config.embedding.model_name, fid),
+                )
+                reindexed += 1
+            except Exception as exc:
+                logger.warning(
+                    "Failed to update embedding for fact %s: %s",
+                    fid[:16], exc,
+                )
+
+    # Update stored signature after successful migration.
+    _write_stored_signature(config.base_dir, current_sig)
+    logger.info(
+        "Embedding migration complete: %d/%d facts re-embedded.",
+        reindexed, total,
+    )
+    return reindexed

package/src/superlocalmemory/storage/migration_v33.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SLM 3.2.3 -> 3.3 migration: idempotent schema upgrade.
+
+Detects a 3.2.3 database (missing 3.3 tables) and creates
+new tables via CREATE TABLE IF NOT EXISTS. Safe to call on
+any DB version — existing tables are never altered.
+
+New tables in 3.3 (all from Phases A-G):
+  - fact_retention           (Phase A: Ebbinghaus forgetting)
+  - polar_embeddings         (Phase B: PolarQuant quantization)
+  - embedding_quantization_metadata  (Phase B: EAP bit-width tracking)
+  - ccq_consolidated_blocks  (Phase E: Cognitive Consolidation)
+  - ccq_audit_log            (Phase E: CCQ audit trail)
+  - soft_prompt_templates    (Phase F: Learning Brain)
+
+All six are already defined in schema_v32.py V32_DDL and created
+by create_all_tables(). This migration module detects their absence
+and creates them for databases that were created before SLM 3.3.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: MIT
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from dataclasses import dataclass, field
+from typing import Final
+
+logger = logging.getLogger(__name__)
+
+
+# Tables introduced or required by SLM 3.3 phases
+_V33_TABLES: Final[tuple[str, ...]] = (
+    "fact_retention",
+    "polar_embeddings",
+    "embedding_quantization_metadata",
+    "ccq_consolidated_blocks",
+    "ccq_audit_log",
+    "soft_prompt_templates",
+)
+
+
+@dataclass(frozen=True)
+class MigrationReport:
+    """Report of migration actions taken."""
+
+    tables_checked: int = 0
+    tables_created: tuple[str, ...] = ()
+    tables_existed: tuple[str, ...] = ()
+    errors: tuple[str, ...] = ()
+
+    @property
+    def is_clean(self) -> bool:
+        return len(self.errors) == 0
+
+
+def _get_existing_tables(conn: sqlite3.Connection) -> frozenset[str]:
+    """Return set of all table names in the database."""
+    rows = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table'"
+    ).fetchall()
+    return frozenset(row[0] for row in rows)
+
+
+def detect_v323_database(conn: sqlite3.Connection) -> bool:
+    """Detect if this is a pre-3.3 database missing new tables.
+
+    Returns True if at least one V33 table is missing.
+    """
+    existing = _get_existing_tables(conn)
+    return any(table not in existing for table in _V33_TABLES)
+
+
+def migrate_v323_to_v33(conn: sqlite3.Connection) -> MigrationReport:
+    """Idempotent migration from SLM 3.2.3 to 3.3.
+
+    Creates missing 3.3 tables via the canonical DDL from schema_v32.py.
+    Uses CREATE TABLE IF NOT EXISTS — safe to run on any DB version.
+
+    Args:
+        conn: Open SQLite connection. Caller manages commit.
+
+    Returns:
+        MigrationReport with details of actions taken.
+    """
+    from superlocalmemory.storage.schema_v32 import V32_DDL
+
+    existing = _get_existing_tables(conn)
+    created: list[str] = []
+    existed: list[str] = []
+    errors: list[str] = []
+
+    for table in _V33_TABLES:
+        if table in existing:
+            existed.append(table)
+            logger.debug("Table %s already exists — skipping", table)
+            continue
+
+        # Find the DDL statement that creates this table
+        ddl_found = False
+        for ddl in V32_DDL:
+            if f"CREATE TABLE IF NOT EXISTS {table}" in ddl:
+                try:
+                    conn.executescript(ddl)
+                    created.append(table)
+                    ddl_found = True
+                    logger.info("Created table: %s", table)
+                except sqlite3.Error as exc:
+                    msg = f"Failed to create {table}: {exc}"
+                    errors.append(msg)
+                    logger.error(msg)
+                    ddl_found = True
+                break
+
+        if not ddl_found:
+            msg = f"No DDL found for table: {table}"
+            errors.append(msg)
+            logger.warning(msg)
+
+    report = MigrationReport(
+        tables_checked=len(_V33_TABLES),
+        tables_created=tuple(created),
+        tables_existed=tuple(existed),
+        errors=tuple(errors),
+    )
+
+    logger.info(
+        "Migration report: checked=%d, created=%d, existed=%d, errors=%d",
+        report.tables_checked,
+        len(report.tables_created),
+        len(report.tables_existed),
+        len(report.errors),
+    )
+
+    return report