superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/storage/migrations.py

@@ -0,0 +1,303 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Database Migrations.
+
+Handles schema versioning and V1 (slm_alpha) → Innovation migration.
+Preserves existing data while upgrading schema structure.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+CURRENT_SCHEMA_VERSION = 1  # Innovation v1
+
+
+def get_schema_version(conn: sqlite3.Connection) -> int:
+    """Read current schema version. Returns 0 if no version table."""
+    try:
+        row = conn.execute(
+            "SELECT version FROM schema_version ORDER BY applied_at DESC LIMIT 1"
+        ).fetchone()
+        return int(row[0]) if row else 0
+    except sqlite3.OperationalError:
+        return 0
+
+
+def set_schema_version(conn: sqlite3.Connection, version: int) -> None:
+    """Record a schema version upgrade."""
+    conn.execute(
+        "INSERT INTO schema_version (version, applied_at, description) VALUES (?, ?, ?)",
+        (version, datetime.now(UTC).isoformat(), f"Migration to v{version}"),
+    )
+
+
+def needs_migration(db_path: Path) -> bool:
+    """Check if database needs migration."""
+    if not db_path.exists():
+        return False
+    conn = sqlite3.connect(str(db_path))
+    try:
+        version = get_schema_version(conn)
+        return version < CURRENT_SCHEMA_VERSION
+    finally:
+        conn.close()
+
+
+def is_v1_database(db_path: Path) -> bool:
+    """Detect if this is a V1 (slm_alpha) database by checking for V1-specific tables."""
+    if not db_path.exists():
+        return False
+    conn = sqlite3.connect(str(db_path))
+    try:
+        tables = {
+            row[0]
+            for row in conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table'"
+            ).fetchall()
+        }
+        # V1 markers: episodic_memory table + no schema_version
+        return "episodic_memory" in tables and "schema_version" not in tables
+    finally:
+        conn.close()
+
+
+def backup_database(db_path: Path) -> Path:
+    """Create timestamped backup before migration."""
+    timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
+    backup_path = db_path.with_suffix(f".backup_{timestamp}.db")
+    shutil.copy2(str(db_path), str(backup_path))
+    logger.info("Database backed up to %s", backup_path)
+    return backup_path
+
+
+def migrate_v1_to_innovation(
+    v1_db_path: Path,
+    innovation_db_path: Path,
+    profile_id: str = "default",
+) -> dict[str, Any]:
+    """Migrate V1 (slm_alpha) database to Innovation schema.
+
+    Preserves all memories and facts. Creates proper typed stores.
+    Returns migration statistics.
+    """
+    from superlocalmemory.storage import schema
+
+    stats: dict[str, Any] = {
+        "memories_migrated": 0,
+        "facts_migrated": 0,
+        "entities_migrated": 0,
+        "edges_migrated": 0,
+        "errors": [],
+    }
+
+    if not v1_db_path.exists():
+        stats["errors"].append(f"V1 database not found: {v1_db_path}")
+        return stats
+
+    # Backup V1 database
+    backup_database(v1_db_path)
+
+    # Create Innovation database with clean schema
+    new_conn = sqlite3.connect(str(innovation_db_path))
+    new_conn.row_factory = sqlite3.Row
+    try:
+        new_conn.execute("PRAGMA foreign_keys=ON")
+        schema.create_all_tables(new_conn)
+
+        # Ensure default profile exists
+        new_conn.execute(
+            "INSERT OR IGNORE INTO profiles (profile_id, name, description, mode, created_at) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (profile_id, profile_id, "Migrated from V1", "a", datetime.now(UTC).isoformat()),
+        )
+
+        # Open V1 database
+        v1_conn = sqlite3.connect(str(v1_db_path))
+        v1_conn.row_factory = sqlite3.Row
+        try:
+            stats = _migrate_memories(v1_conn, new_conn, profile_id, stats)
+            stats = _migrate_facts(v1_conn, new_conn, profile_id, stats)
+            stats = _migrate_entities(v1_conn, new_conn, profile_id, stats)
+            stats = _migrate_edges(v1_conn, new_conn, profile_id, stats)
+            new_conn.commit()
+        finally:
+            v1_conn.close()
+
+        set_schema_version(new_conn, CURRENT_SCHEMA_VERSION)
+        new_conn.commit()
+        logger.info("Migration complete: %s", stats)
+
+    finally:
+        new_conn.close()
+
+    return stats
+
+
+def _migrate_memories(
+    v1: sqlite3.Connection,
+    new: sqlite3.Connection,
+    profile_id: str,
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """Migrate episodic_memory → memories table."""
+    try:
+        rows = v1.execute("SELECT * FROM episodic_memory").fetchall()
+    except sqlite3.OperationalError:
+        stats["errors"].append("No episodic_memory table in V1")
+        return stats
+
+    for row in rows:
+        d = dict(row)
+        try:
+            new.execute(
+                "INSERT OR IGNORE INTO memories "
+                "(memory_id, profile_id, content, session_id, speaker, role, "
+                "session_date, created_at, metadata) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    d.get("memory_id", d.get("id", "")),
+                    profile_id,
+                    d.get("content", ""),
+                    d.get("session_id", ""),
+                    d.get("speaker", ""),
+                    d.get("role", ""),
+                    d.get("session_date"),
+                    d.get("created_at", datetime.now(UTC).isoformat()),
+                    json.dumps({"migrated_from": "v1"}),
+                ),
+            )
+            stats["memories_migrated"] += 1
+        except sqlite3.Error as exc:
+            stats["errors"].append(f"Memory {d.get('id', '?')}: {exc}")
+
+    return stats
+
+
+def _migrate_facts(
+    v1: sqlite3.Connection,
+    new: sqlite3.Connection,
+    profile_id: str,
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """Migrate semantic_facts → facts table."""
+    try:
+        rows = v1.execute("SELECT * FROM semantic_facts").fetchall()
+    except sqlite3.OperationalError:
+        return stats
+
+    for row in rows:
+        d = dict(row)
+        try:
+            new.execute(
+                "INSERT OR IGNORE INTO facts "
+                "(fact_id, memory_id, profile_id, content, fact_type, "
+                "entities, canonical_entities, confidence, importance, "
+                "evidence_count, access_count, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    d.get("fact_id", d.get("id", "")),
+                    d.get("memory_id", ""),
+                    profile_id,
+                    d.get("content", d.get("fact", "")),
+                    d.get("fact_type", "semantic"),
+                    d.get("entities_json", "[]"),
+                    d.get("canonical_entities_json", "[]"),
+                    d.get("confidence", 1.0),
+                    d.get("importance", 0.5),
+                    d.get("evidence_count", 1),
+                    d.get("access_count", 0),
+                    d.get("created_at", datetime.now(UTC).isoformat()),
+                ),
+            )
+            stats["facts_migrated"] += 1
+        except sqlite3.Error as exc:
+            stats["errors"].append(f"Fact {d.get('id', '?')}: {exc}")
+
+    return stats
+
+
+def _migrate_entities(
+    v1: sqlite3.Connection,
+    new: sqlite3.Connection,
+    profile_id: str,
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """Migrate canonical_entities → entities table."""
+    try:
+        rows = v1.execute("SELECT * FROM canonical_entities").fetchall()
+    except sqlite3.OperationalError:
+        return stats
+
+    for row in rows:
+        d = dict(row)
+        try:
+            new.execute(
+                "INSERT OR IGNORE INTO entities "
+                "(entity_id, profile_id, canonical_name, entity_type, "
+                "first_seen, last_seen, fact_count) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (
+                    d.get("entity_id", d.get("id", "")),
+                    profile_id,
+                    d.get("canonical_name", d.get("name", "")),
+                    d.get("entity_type", "unknown"),
+                    d.get("first_seen", datetime.now(UTC).isoformat()),
+                    d.get("last_seen", datetime.now(UTC).isoformat()),
+                    d.get("fact_count", 0),
+                ),
+            )
+            stats["entities_migrated"] += 1
+        except sqlite3.Error as exc:
+            stats["errors"].append(f"Entity {d.get('id', '?')}: {exc}")
+
+    return stats
+
+
+def _migrate_edges(
+    v1: sqlite3.Connection,
+    new: sqlite3.Connection,
+    profile_id: str,
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """Migrate memory_edges → graph_edges table."""
+    try:
+        rows = v1.execute("SELECT * FROM memory_edges").fetchall()
+    except sqlite3.OperationalError:
+        return stats
+
+    for row in rows:
+        d = dict(row)
+        try:
+            new.execute(
+                "INSERT OR IGNORE INTO graph_edges "
+                "(edge_id, profile_id, source_id, target_id, edge_type, "
+                "weight, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                (
+                    d.get("edge_id", d.get("id", "")),
+                    profile_id,
+                    d.get("source_id", d.get("source", "")),
+                    d.get("target_id", d.get("target", "")),
+                    d.get("edge_type", d.get("relation_type", "entity")),
+                    d.get("weight", 1.0),
+                    d.get("created_at", datetime.now(UTC).isoformat()),
+                ),
+            )
+            stats["edges_migrated"] += 1
+        except sqlite3.Error as exc:
+            stats["errors"].append(f"Edge {d.get('id', '?')}: {exc}")
+
+    return stats

package/src/superlocalmemory/storage/models.py

@@ -0,0 +1,406 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Data Models.
+
+All dataclasses for the memory system. Profile-scoped by design.
+Every model that persists to DB includes profile_id.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import Enum
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+class FactType(str, Enum):
+    """Memory fact classification — ENGRAM-inspired typed stores."""
+
+    EPISODIC = "episodic"      # Events: who did what when
+    SEMANTIC = "semantic"      # World knowledge: X is Y
+    OPINION = "opinion"        # Subjective with confidence
+    TEMPORAL = "temporal"      # Time-bounded events with intervals
+
+
+class EdgeType(str, Enum):
+    """Knowledge graph edge types."""
+
+    ENTITY = "entity"          # Shared entity link
+    TEMPORAL = "temporal"      # Chronological ordering
+    SEMANTIC = "semantic"      # Embedding similarity
+    CAUSAL = "causal"          # Cause-effect relationship
+    CONTRADICTION = "contradiction"  # Conflicting facts
+    SUPERSEDES = "supersedes"  # Newer fact replaces older
+
+
+class ConsolidationActionType(str, Enum):
+    """Mem0-style consolidation actions."""
+
+    ADD = "add"                # New information, store as new fact
+    UPDATE = "update"          # Refines existing fact
+    SUPERSEDE = "supersede"    # Contradicts existing fact
+    NOOP = "noop"              # Duplicate, skip storage
+
+
+class MemoryLifecycle(str, Enum):
+    """Memory lifecycle states — coupled with Langevin dynamics."""
+
+    ACTIVE = "active"          # Frequently accessed, near origin
+    WARM = "warm"              # Moderate access, mid-radius
+    COLD = "cold"              # Rarely accessed, near boundary
+    ARCHIVED = "archived"      # Beyond boundary, deep storage
+
+
+class SignalType(str, Enum):
+    """V2-compatible signal inference types."""
+
+    FACTUAL = "factual"
+    EMOTIONAL = "emotional"
+    TEMPORAL = "temporal"
+    OPINION = "opinion"
+    REQUEST = "request"
+    SOCIAL = "social"
+
+
+class Mode(str, Enum):
+    """Operating modes — EU AI Act alignment."""
+
+    A = "a"  # Local Guardian: zero LLM, EU AI Act FULL
+    B = "b"  # Smart Local: local Ollama LLM, EU AI Act FULL
+    C = "c"  # Full Power: UNRESTRICTED, best models, 90%+ target
+
+
+# ---------------------------------------------------------------------------
+# Core Models
+# ---------------------------------------------------------------------------
+
+def _new_id() -> str:
+    return uuid.uuid4().hex[:16]
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+@dataclass(frozen=True)
+class Profile:
+    """Memory profile — first-class isolation boundary.
+
+    Every table is scoped by profile_id. Instant switching.
+    Ported from V2.8 columnar profile isolation pattern.
+    """
+
+    profile_id: str
+    name: str
+    description: str = ""
+    personality: str = ""          # Profile personality description
+    mode: Mode = Mode.A            # Default operating mode for this profile
+    created_at: str = field(default_factory=_now)
+    last_used: str | None = None
+    config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class MemoryRecord:
+    """Primary memory entry — a conversation turn or message.
+
+    This is the RAW input. Facts are extracted FROM this during encoding.
+    """
+
+    memory_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    content: str = ""
+    session_id: str = ""
+    speaker: str = ""              # Who said this
+    role: str = ""                 # user / assistant / system
+    session_date: str | None = None   # Parsed ISO-8601 date
+    created_at: str = field(default_factory=_now)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class AtomicFact:
+    """Structured fact extracted from memory — the PRIMARY retrieval unit.
+
+    This is what the encoding engine produces. This is what retrieval searches.
+    Typed stores (ENGRAM pattern): episodic / semantic / opinion / temporal.
+    """
+
+    fact_id: str = field(default_factory=_new_id)
+    memory_id: str = ""            # Source memory this was extracted from
+    profile_id: str = "default"
+    content: str = ""              # Atomic fact statement
+    fact_type: FactType = FactType.SEMANTIC
+
+    # Entities
+    entities: list[str] = field(default_factory=list)
+    canonical_entities: list[str] = field(default_factory=list)
+
+    # Temporal (3-date model — Mastra pattern)
+    observation_date: str | None = None    # When the conversation happened
+    referenced_date: str | None = None     # Date mentioned in content
+    interval_start: str | None = None      # Event start (for duration events)
+    interval_end: str | None = None        # Event end
+
+    # Quality
+    confidence: float = 1.0
+    importance: float = 0.5        # Priority: 0.0 (low) to 1.0 (critical)
+    evidence_count: int = 1        # Incremented on UPDATE consolidation
+    access_count: int = 0
+
+    # Source tracing
+    source_turn_ids: list[str] = field(default_factory=list)
+    session_id: str = ""
+
+    # Embeddings (populated by encoding pipeline)
+    embedding: list[float] | None = None
+    fisher_mean: list[float] | None = None
+    fisher_variance: list[float] | None = None
+
+    # Lifecycle (Langevin-coupled)
+    lifecycle: MemoryLifecycle = MemoryLifecycle.ACTIVE
+    langevin_position: list[float] | None = None
+
+    # Emotional (VADER)
+    emotional_valence: float = 0.0   # -1.0 negative to +1.0 positive
+    emotional_arousal: float = 0.0   # 0.0 calm to 1.0 intense
+
+    # Signal type (V2 compatible)
+    signal_type: SignalType = SignalType.FACTUAL
+
+    created_at: str = field(default_factory=_now)
+
+
+@dataclass
+class CanonicalEntity:
+    """Resolved canonical entity — the single identity for an entity.
+
+    Aliases point here. Entity profiles accumulate knowledge about this entity.
+    """
+
+    entity_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    canonical_name: str = ""
+    entity_type: str = ""          # person / place / org / concept / event
+    first_seen: str = field(default_factory=_now)
+    last_seen: str = field(default_factory=_now)
+    fact_count: int = 0
+
+
+@dataclass
+class EntityAlias:
+    """Alias mapping to a canonical entity."""
+
+    alias_id: str = field(default_factory=_new_id)
+    entity_id: str = ""            # FK to CanonicalEntity
+    alias: str = ""
+    confidence: float = 1.0
+    source: str = ""               # How this alias was discovered
+
+
+@dataclass
+class EntityProfile:
+    """Accumulated knowledge about a canonical entity.
+
+    Built during encoding by ObservationBuilder. Updated on each new fact.
+    """
+
+    profile_entry_id: str = field(default_factory=_new_id)
+    entity_id: str = ""            # FK to CanonicalEntity
+    profile_id: str = "default"
+    knowledge_summary: str = ""    # Running summary of all facts about entity
+    fact_ids: list[str] = field(default_factory=list)
+    last_updated: str = field(default_factory=_now)
+
+
+@dataclass
+class MemoryScene:
+    """Clustered group of related facts — EverMemOS MemScene pattern.
+
+    Scenes provide contextual retrieval: related facts come together.
+    """
+
+    scene_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    theme: str = ""                # Scene description / topic
+    fact_ids: list[str] = field(default_factory=list)
+    entity_ids: list[str] = field(default_factory=list)
+    created_at: str = field(default_factory=_now)
+    last_updated: str = field(default_factory=_now)
+
+
+@dataclass
+class TemporalEvent:
+    """Per-entity timeline entry with 3-date model.
+
+    Enables temporal retrieval: "What happened to Alice in March?"
+    """
+
+    event_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    entity_id: str = ""            # FK to CanonicalEntity
+    fact_id: str = ""              # FK to AtomicFact
+    observation_date: str | None = None
+    referenced_date: str | None = None
+    interval_start: str | None = None
+    interval_end: str | None = None
+    description: str = ""
+
+
+@dataclass
+class GraphEdge:
+    """Knowledge graph edge between facts or entities."""
+
+    edge_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    source_id: str = ""            # Fact ID or Entity ID
+    target_id: str = ""            # Fact ID or Entity ID
+    edge_type: EdgeType = EdgeType.ENTITY
+    weight: float = 1.0
+    created_at: str = field(default_factory=_now)
+
+
+@dataclass
+class ConsolidationAction:
+    """Log of consolidation decisions (ADD/UPDATE/SUPERSEDE/NOOP)."""
+
+    action_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    action_type: ConsolidationActionType = ConsolidationActionType.ADD
+    new_fact_id: str = ""          # The incoming fact
+    existing_fact_id: str = ""     # The matched existing fact (if any)
+    reason: str = ""               # Why this action was chosen
+    timestamp: str = field(default_factory=_now)
+
+
+# ---------------------------------------------------------------------------
+# Trust & Provenance (Ported from V2.8)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class TrustScore:
+    """Bayesian trust score per source / entity / fact.
+
+    Updated on access, contradiction, confirmation. Profile-scoped.
+    """
+
+    trust_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    target_type: str = ""          # "entity" / "source" / "fact"
+    target_id: str = ""            # ID of the entity/source/fact
+    trust_score: float = 0.5       # Bayesian: 0.0 = untrusted, 1.0 = fully trusted
+    evidence_count: int = 0
+    last_updated: str = field(default_factory=_now)
+
+
+@dataclass
+class ProvenanceRecord:
+    """Provenance tracking — who/what created this memory and how."""
+
+    provenance_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    fact_id: str = ""
+    source_type: str = ""          # "conversation" / "import" / "consolidation"
+    source_id: str = ""            # Session ID or import batch ID
+    created_by: str = ""           # Agent or user who created this
+    timestamp: str = field(default_factory=_now)
+
+
+# ---------------------------------------------------------------------------
+# Learning & Behavioral (Ported from V2.8)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class FeedbackRecord:
+    """User feedback on retrieval results — drives adaptive learning."""
+
+    feedback_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    query: str = ""
+    fact_id: str = ""
+    feedback_type: str = ""        # "relevant" / "irrelevant" / "partial"
+    dwell_time_ms: int = 0
+    timestamp: str = field(default_factory=_now)
+
+
+@dataclass
+class BehavioralPattern:
+    """Learned behavioral pattern — query habits, topic preferences."""
+
+    pattern_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    pattern_type: str = ""         # "query_topic" / "time_of_day" / "entity_pref"
+    pattern_key: str = ""          # The pattern identifier
+    pattern_value: str = ""        # Serialized pattern data
+    confidence: float = 0.0
+    observation_count: int = 0
+    last_updated: str = field(default_factory=_now)
+
+
+@dataclass
+class ActionOutcome:
+    """Outcome tracking for learning — did the retrieved facts help?"""
+
+    outcome_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    query: str = ""
+    fact_ids: list[str] = field(default_factory=list)
+    outcome: str = ""              # "success" / "partial" / "failure"
+    context: dict[str, Any] = field(default_factory=dict)
+    timestamp: str = field(default_factory=_now)
+
+
+# ---------------------------------------------------------------------------
+# Compliance (EU AI Act, GDPR, Lifecycle)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ComplianceAuditEntry:
+    """Audit trail for compliance (GDPR right-to-know, EU AI Act)."""
+
+    audit_id: str = field(default_factory=_new_id)
+    profile_id: str = "default"
+    action: str = ""               # "store" / "retrieve" / "delete" / "export"
+    target_type: str = ""          # "fact" / "memory" / "profile"
+    target_id: str = ""
+    details: str = ""
+    timestamp: str = field(default_factory=_now)
+
+
+# ---------------------------------------------------------------------------
+# Retrieval Result (Not persisted — runtime only)
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RetrievalResult:
+    """A single retrieval result with scores and evidence chain."""
+
+    fact: AtomicFact
+    score: float = 0.0
+    channel_scores: dict[str, float] = field(default_factory=dict)
+    confidence: float = 0.0
+    evidence_chain: list[str] = field(default_factory=list)
+    trust_score: float = 0.5
+
+
+@dataclass
+class RecallResponse:
+    """Complete recall response with ranked results and metadata."""
+
+    query: str = ""
+    mode: Mode = Mode.A
+    results: list[RetrievalResult] = field(default_factory=list)
+    query_type: str = ""           # factual / temporal / opinion / multi-hop
+    channel_weights: dict[str, float] = field(default_factory=dict)
+    total_candidates: int = 0
+    retrieval_time_ms: float = 0.0