superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/storage/schema_v3410.py

@@ -1,159 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3.4.10 "Fortress" — Schema Extensions.
-
-New tables for cloud backup + entity quality:
-  - backup_destinations: Cloud backup targets (Google Drive, GitHub, local)
-  - entity_blacklist: Stop words and known garbage entity names
-
-Design rules (inherited):
-  - CREATE IF NOT EXISTS for idempotency
-  - profile_id where applicable
-  - Never ALTER existing column types
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# DDL — Backup Destinations
-# ---------------------------------------------------------------------------
-
-_BACKUP_DESTINATIONS_DDL = """
-CREATE TABLE IF NOT EXISTS backup_destinations (
-    id TEXT PRIMARY KEY,
-    profile_id TEXT DEFAULT 'default',
-    destination_type TEXT NOT NULL,
-    display_name TEXT DEFAULT '',
-    credentials_ref TEXT DEFAULT '',
-    config TEXT DEFAULT '{}',
-    last_sync_at TEXT,
-    last_sync_status TEXT DEFAULT 'never',
-    last_sync_error TEXT DEFAULT '',
-    enabled INTEGER DEFAULT 1,
-    created_at TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS idx_backup_dest_type
-    ON backup_destinations(destination_type);
-CREATE INDEX IF NOT EXISTS idx_backup_dest_profile
-    ON backup_destinations(profile_id);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Entity Blacklist
-# ---------------------------------------------------------------------------
-
-_ENTITY_BLACKLIST_DDL = """
-CREATE TABLE IF NOT EXISTS entity_blacklist (
-    term TEXT PRIMARY KEY,
-    reason TEXT DEFAULT 'stop_word',
-    added_at TEXT NOT NULL
-);
-"""
-
-# ---------------------------------------------------------------------------
-# Default blacklist entries (seeded on first migration)
-# ---------------------------------------------------------------------------
-
-_DEFAULT_BLACKLIST = [
-    # English stop words that frequently become garbage entities
-    "a", "an", "the", "all", "not", "no", "yes", "and", "or", "but",
-    "if", "is", "are", "was", "were", "be", "been", "being",
-    "have", "has", "had", "do", "does", "did", "will", "would",
-    "shall", "should", "can", "could", "just", "also", "only",
-    "very", "too", "so", "then", "than", "that", "this",
-    "each", "every", "both", "few", "more", "most", "other",
-    "some", "such", "any", "many", "much", "own", "same",
-    "new", "old", "first", "last", "next", "now",
-    # Months (biggest historical source of garbage)
-    "january", "february", "march", "april", "may", "june",
-    "july", "august", "september", "october", "november", "december",
-    # Technical terms commonly misclassified
-    "test", "fix", "build", "check", "run", "start", "stop",
-    "error", "status", "version", "query", "data", "file",
-    "ready", "done", "complete", "pending", "active", "failed",
-    "total", "count", "key", "value", "true", "false",
-    # Abstract nouns misclassified as people
-    "completeness", "correctness", "limitations", "requirements",
-    "performance", "security", "quality", "coverage", "progress",
-    "analysis", "research", "implementation", "verification",
-]
-
-
-# ---------------------------------------------------------------------------
-# Migration runner
-# ---------------------------------------------------------------------------
-
-def apply_v3410_schema(db_path: str | sqlite3.Connection) -> dict:
-    """Apply all v3.4.10 schema changes. Idempotent."""
-    result = {"applied": [], "errors": []}
-
-    if isinstance(db_path, sqlite3.Connection):
-        conn = db_path
-        own_connection = False
-    else:
-        conn = sqlite3.connect(str(db_path))
-        own_connection = True
-
-    try:
-        # Backup destinations table
-        try:
-            conn.executescript(_BACKUP_DESTINATIONS_DDL)
-            result["applied"].append("backup_destinations table + indexes")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"backup_destinations: {e}")
-
-        # Entity blacklist table
-        try:
-            conn.executescript(_ENTITY_BLACKLIST_DDL)
-            result["applied"].append("entity_blacklist table")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"entity_blacklist: {e}")
-
-        # Seed default blacklist (only inserts missing entries)
-        now = __import__("datetime").datetime.now().isoformat()
-        seeded = 0
-        for term in _DEFAULT_BLACKLIST:
-            try:
-                conn.execute(
-                    "INSERT OR IGNORE INTO entity_blacklist (term, reason, added_at) "
-                    "VALUES (?, 'stop_word', ?)",
-                    (term, now),
-                )
-                seeded += 1
-            except sqlite3.OperationalError:
-                pass
-        if seeded:
-            result["applied"].append(f"entity_blacklist seeded ({len(_DEFAULT_BLACKLIST)} terms)")
-
-        # Mark version
-        try:
-            conn.execute(
-                "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
-                ("3.4.10", now),
-            )
-        except sqlite3.OperationalError:
-            pass
-
-        conn.commit()
-
-        if result["applied"]:
-            logger.info("Schema v3.4.10 applied: %s", ", ".join(result["applied"]))
-
-    except Exception as e:
-        result["errors"].append(f"fatal: {e}")
-        logger.error("Schema v3.4.10 migration failed: %s", e)
-    finally:
-        if own_connection:
-            conn.close()
-
-    return result

package/src/superlocalmemory/storage/schema_v3411.py

@@ -1,149 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3.4.11 "Scale-Ready" — Schema Extensions.
-
-New tables for tiered storage + future backend integrations:
-  - pinned_facts: User-pinned facts that stay in hot tier forever
-  - backend_status: Tracks initialization state of LanceDB/KùzuDB backends
-  - fact_consolidations: History of merged/consolidated facts
-
-Design rules (inherited):
-  - CREATE IF NOT EXISTS for idempotency
-  - profile_id where applicable
-  - Never ALTER existing column types
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# DDL — Pinned Facts (user override on lifecycle demotion)
-# ---------------------------------------------------------------------------
-
-_PINNED_FACTS_DDL = """
-CREATE TABLE IF NOT EXISTS pinned_facts (
-    fact_id TEXT PRIMARY KEY,
-    profile_id TEXT DEFAULT 'default',
-    pinned_at TEXT NOT NULL,
-    reason TEXT DEFAULT ''
-);
-
-CREATE INDEX IF NOT EXISTS idx_pinned_facts_profile
-    ON pinned_facts(profile_id);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Backend Status (LanceDB, KùzuDB, sqlite-vec tracking)
-# ---------------------------------------------------------------------------
-
-_BACKEND_STATUS_DDL = """
-CREATE TABLE IF NOT EXISTS backend_status (
-    backend_name TEXT PRIMARY KEY,
-    status TEXT DEFAULT 'not_initialized',
-    record_count INTEGER DEFAULT 0,
-    last_sync_at TEXT,
-    error_message TEXT DEFAULT '',
-    config TEXT DEFAULT '{}'
-);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Fact Consolidations (merge history)
-# ---------------------------------------------------------------------------
-
-_FACT_CONSOLIDATIONS_DDL = """
-CREATE TABLE IF NOT EXISTS fact_consolidations (
-    consolidation_id TEXT PRIMARY KEY,
-    profile_id TEXT DEFAULT 'default',
-    consolidated_fact_id TEXT NOT NULL,
-    source_fact_ids TEXT NOT NULL,
-    strategy TEXT DEFAULT 'entity_cluster',
-    created_at TEXT NOT NULL
-);
-
-CREATE INDEX IF NOT EXISTS idx_fact_consolidations_profile
-    ON fact_consolidations(profile_id);
-CREATE INDEX IF NOT EXISTS idx_fact_consolidations_target
-    ON fact_consolidations(consolidated_fact_id);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Index on lifecycle for fast tier queries
-# ---------------------------------------------------------------------------
-
-_LIFECYCLE_INDEX_DDL = """
-CREATE INDEX IF NOT EXISTS idx_atomic_facts_lifecycle
-    ON atomic_facts(lifecycle, profile_id);
-"""
-
-
-# ---------------------------------------------------------------------------
-# Migration runner
-# ---------------------------------------------------------------------------
-
-def apply_v3411_schema(db_path: str | sqlite3.Connection) -> dict:
-    """Apply all v3.4.11 schema changes. Idempotent."""
-    result = {"applied": [], "errors": []}
-
-    if isinstance(db_path, sqlite3.Connection):
-        conn = db_path
-        own_connection = False
-    else:
-        conn = sqlite3.connect(str(db_path))
-        own_connection = True
-
-    try:
-        for name, ddl in [
-            ("pinned_facts", _PINNED_FACTS_DDL),
-            ("backend_status", _BACKEND_STATUS_DDL),
-            ("fact_consolidations", _FACT_CONSOLIDATIONS_DDL),
-            ("lifecycle_index", _LIFECYCLE_INDEX_DDL),
-        ]:
-            try:
-                conn.executescript(ddl)
-                result["applied"].append(name)
-            except sqlite3.OperationalError as e:
-                result["errors"].append(f"{name}: {e}")
-
-        # Seed default backend entries
-        now = __import__("datetime").datetime.now().isoformat()
-        for backend in ("sqlite_vec", "sqlite_graph", "lancedb", "kuzu"):
-            try:
-                conn.execute(
-                    "INSERT OR IGNORE INTO backend_status "
-                    "(backend_name, status, config) VALUES (?, ?, ?)",
-                    (backend, "not_initialized" if backend in ("lancedb", "kuzu") else "active", "{}"),
-                )
-            except sqlite3.OperationalError:
-                pass
-
-        # Mark version
-        try:
-            conn.execute(
-                "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
-                ("3.4.11", now),
-            )
-        except sqlite3.OperationalError:
-            pass
-
-        conn.commit()
-
-        if result["applied"]:
-            logger.info("Schema v3.4.11 applied: %s", ", ".join(result["applied"]))
-
-    except Exception as e:
-        result["errors"].append(f"fatal: {e}")
-        logger.error("Schema v3.4.11 migration failed: %s", e)
-    finally:
-        if own_connection:
-            conn.close()
-
-    return result

package/src/superlocalmemory/storage/schema_v343.py

@@ -1,315 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3.4.3 "The Unified Brain" — Schema Extensions.
-
-All new tables for v3.4.3 features:
-  - Phase C: Mesh broker tables (mesh_peers, mesh_messages, mesh_state, mesh_locks, mesh_events)
-  - Phase D: Entity compilation (ALTER entity_profiles + new index)
-  - Phase E: Ingestion log (ingestion_log)
-
-Design rules (inherited from schema_v32.py):
-  - CREATE IF NOT EXISTS for idempotency
-  - profile_id where applicable
-  - Never ALTER existing column types (add new columns only)
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from typing import Final
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# Table names (for reference and rollback)
-# ---------------------------------------------------------------------------
-
-V343_TABLES: Final[tuple[str, ...]] = (
-    "mesh_peers",
-    "mesh_messages",
-    "mesh_state",
-    "mesh_locks",
-    "mesh_events",
-    "ingestion_log",
-)
-
-# ---------------------------------------------------------------------------
-# DDL — Phase C: Mesh Broker Tables
-# ---------------------------------------------------------------------------
-
-_MESH_DDL = """
--- Mesh Peers  (profile_id = tenant boundary; see M023)
-CREATE TABLE IF NOT EXISTS mesh_peers (
-    peer_id TEXT PRIMARY KEY,
-    session_id TEXT NOT NULL,
-    summary TEXT DEFAULT '',
-    status TEXT DEFAULT 'active',
-    host TEXT DEFAULT '127.0.0.1',
-    port INTEGER DEFAULT 0,
-    registered_at TEXT NOT NULL,
-    last_heartbeat TEXT NOT NULL,
-    profile_id TEXT NOT NULL DEFAULT 'default'
-);
-
--- Mesh Messages
-CREATE TABLE IF NOT EXISTS mesh_messages (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    from_peer TEXT NOT NULL,
-    to_peer TEXT NOT NULL,
-    msg_type TEXT DEFAULT 'text',
-    content TEXT NOT NULL,
-    read INTEGER DEFAULT 0,
-    created_at TEXT NOT NULL,
-    profile_id TEXT NOT NULL DEFAULT 'default'
-);
-
--- Mesh State (shared key-value store, per tenant)
-CREATE TABLE IF NOT EXISTS mesh_state (
-    profile_id TEXT NOT NULL DEFAULT 'default',
-    key TEXT NOT NULL,
-    value TEXT NOT NULL,
-    set_by TEXT NOT NULL,
-    updated_at TEXT NOT NULL,
-    PRIMARY KEY (profile_id, key)
-);
-
--- Mesh Locks (file-level locks for coordination, per tenant)
-CREATE TABLE IF NOT EXISTS mesh_locks (
-    profile_id TEXT NOT NULL DEFAULT 'default',
-    file_path TEXT NOT NULL,
-    locked_by TEXT NOT NULL,
-    locked_at TEXT NOT NULL,
-    expires_at TEXT NOT NULL DEFAULT '9999-12-31T23:59:59Z',
-    PRIMARY KEY (profile_id, file_path)
-);
-
--- Mesh Events (audit log)
-CREATE TABLE IF NOT EXISTS mesh_events (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    event_type TEXT NOT NULL,
-    payload TEXT DEFAULT '{}',
-    emitted_by TEXT NOT NULL,
-    created_at TEXT NOT NULL,
-    profile_id TEXT NOT NULL DEFAULT 'default'
-);
-
--- NOTE: profile-leading indexes live in migration M023, never here — on an
--- upgrade the profile_id column does not exist when this DDL runs (the old
--- table already exists, so CREATE TABLE IF NOT EXISTS no-ops), and a
--- CREATE INDEX on the missing column would abort engine init.
-CREATE INDEX IF NOT EXISTS idx_mesh_messages_to
-    ON mesh_messages(to_peer, read);
-CREATE INDEX IF NOT EXISTS idx_mesh_events_type
-    ON mesh_events(event_type, created_at);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Phase D: Entity Compilation (ALTER entity_profiles)
-# ---------------------------------------------------------------------------
-
-_ENTITY_COMPILATION_ALTERS = [
-    "ALTER TABLE entity_profiles ADD COLUMN project_name TEXT DEFAULT ''",
-    "ALTER TABLE entity_profiles ADD COLUMN compiled_truth TEXT DEFAULT ''",
-    "ALTER TABLE entity_profiles ADD COLUMN timeline TEXT DEFAULT '[]'",
-    "ALTER TABLE entity_profiles ADD COLUMN compilation_confidence REAL DEFAULT 0.5",
-    "ALTER TABLE entity_profiles ADD COLUMN last_compiled_at TEXT DEFAULT NULL",
-]
-
-_ENTITY_COMPILATION_INDEX = """
-CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_profiles_compilation
-    ON entity_profiles(entity_id, profile_id, project_name);
-CREATE INDEX IF NOT EXISTS idx_entity_profiles_project
-    ON entity_profiles(profile_id, project_name);
-"""
-
-# ---------------------------------------------------------------------------
-# DDL — Phase E: Ingestion Log
-# ---------------------------------------------------------------------------
-
-_INGESTION_DDL = """
-CREATE TABLE IF NOT EXISTS ingestion_log (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    profile_id TEXT NOT NULL DEFAULT 'default',
-    source_type TEXT NOT NULL,
-    dedup_key TEXT NOT NULL,
-    fact_ids TEXT DEFAULT '[]',
-    metadata TEXT DEFAULT '{}',
-    status TEXT DEFAULT 'ingested',
-    ingested_at TEXT NOT NULL,
-    UNIQUE(profile_id, source_type, dedup_key)
-);
-CREATE INDEX IF NOT EXISTS idx_ingestion_dedup
-    ON ingestion_log(profile_id, source_type, dedup_key);
-"""
-
-# ---------------------------------------------------------------------------
-# Version marker
-# ---------------------------------------------------------------------------
-
-_VERSION_DDL = """
-CREATE TABLE IF NOT EXISTS schema_version (
-    version TEXT PRIMARY KEY,
-    applied_at TEXT NOT NULL
-);
-"""
-
-
-# ---------------------------------------------------------------------------
-# Migration runner
-# ---------------------------------------------------------------------------
-
-# ---------------------------------------------------------------------------
-# DDL — v3.4.6: Mesh Connected Brain (broadcast, project routing, offline queue)
-# ---------------------------------------------------------------------------
-
-_MESH_V346_ALTERS = [
-    "ALTER TABLE mesh_peers ADD COLUMN project_path TEXT DEFAULT ''",
-    "ALTER TABLE mesh_peers ADD COLUMN agent_type TEXT DEFAULT 'unknown'",
-    "ALTER TABLE mesh_messages ADD COLUMN expires_at TEXT DEFAULT NULL",
-    "ALTER TABLE mesh_messages ADD COLUMN target_type TEXT DEFAULT 'peer'",
-    "ALTER TABLE mesh_messages ADD COLUMN project_path TEXT DEFAULT ''",
-]
-
-_MESH_V346_DDL = """
--- Read tracking for broadcast/project messages (each peer tracks own reads)
-CREATE TABLE IF NOT EXISTS mesh_reads (
-    message_id INTEGER NOT NULL,
-    peer_id TEXT NOT NULL,
-    read_at TEXT NOT NULL,
-    PRIMARY KEY (message_id, peer_id)
-);
-
-CREATE INDEX IF NOT EXISTS idx_mesh_messages_target
-    ON mesh_messages(target_type, project_path);
-CREATE INDEX IF NOT EXISTS idx_mesh_messages_expires
-    ON mesh_messages(expires_at);
-CREATE INDEX IF NOT EXISTS idx_mesh_peers_project
-    ON mesh_peers(project_path);
-"""
-
-
-def apply_v346_schema(db_path: str | sqlite3.Connection) -> dict:
-    """Apply v3.4.6 mesh enhancements. Idempotent."""
-    result = {"applied": [], "errors": []}
-
-    if isinstance(db_path, sqlite3.Connection):
-        conn = db_path
-        own_connection = False
-    else:
-        conn = sqlite3.connect(str(db_path))
-        own_connection = True
-
-    try:
-        for alter_sql in _MESH_V346_ALTERS:
-            try:
-                conn.execute(alter_sql)
-                col_name = alter_sql.split("ADD COLUMN")[1].strip().split()[0]
-                result["applied"].append(col_name)
-            except sqlite3.OperationalError:
-                pass  # Column already exists
-
-        try:
-            conn.executescript(_MESH_V346_DDL)
-            result["applied"].append("mesh_reads table + indexes")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"mesh_v346: {e}")
-
-        conn.execute(
-            "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
-            ("3.4.6", __import__("datetime").datetime.now().isoformat()),
-        )
-        conn.commit()
-
-        if result["applied"]:
-            logger.info("Schema v3.4.6 applied: %s", ", ".join(result["applied"]))
-
-    except Exception as e:
-        result["errors"].append(f"fatal: {e}")
-    finally:
-        if own_connection:
-            conn.close()
-
-    return result
-
-
-def apply_v343_schema(db_path: str | sqlite3.Connection) -> dict:
-    """Apply all v3.4.3 schema changes. Idempotent — safe to call multiple times.
-
-    Returns dict with migration status and any errors.
-    """
-    result = {"applied": [], "skipped": [], "errors": []}
-
-    if isinstance(db_path, sqlite3.Connection):
-        conn = db_path
-        own_connection = False
-    else:
-        conn = sqlite3.connect(str(db_path))
-        own_connection = True
-
-    try:
-        # Check if already applied
-        try:
-            row = conn.execute(
-                "SELECT version FROM schema_version WHERE version = '3.4.3'"
-            ).fetchone()
-            if row:
-                result["skipped"].append("v3.4.3 already applied")
-                return result
-        except sqlite3.OperationalError:
-            pass  # schema_version table doesn't exist yet
-
-        # Apply version table
-        conn.executescript(_VERSION_DDL)
-
-        # Phase C: Mesh tables
-        try:
-            conn.executescript(_MESH_DDL)
-            result["applied"].append("mesh tables (5 tables, 2 indexes)")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"mesh: {e}")
-
-        # Phase D: Entity compilation ALTER TABLE
-        for alter_sql in _ENTITY_COMPILATION_ALTERS:
-            try:
-                conn.execute(alter_sql)
-                result["applied"].append(alter_sql.split("ADD COLUMN")[1].strip().split()[0])
-            except sqlite3.OperationalError:
-                # Column already exists — fine (idempotent)
-                pass
-
-        try:
-            conn.executescript(_ENTITY_COMPILATION_INDEX)
-            result["applied"].append("entity_profiles indexes")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"entity indexes: {e}")
-
-        # Phase E: Ingestion log
-        try:
-            conn.executescript(_INGESTION_DDL)
-            result["applied"].append("ingestion_log table")
-        except sqlite3.OperationalError as e:
-            result["errors"].append(f"ingestion: {e}")
-
-        # Mark version
-        conn.execute(
-            "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)",
-            ("3.4.3", __import__("datetime").datetime.now().isoformat()),
-        )
-        conn.commit()
-
-        if result["applied"]:
-            logger.info("Schema v3.4.3 applied: %s", ", ".join(result["applied"]))
-
-    except Exception as e:
-        result["errors"].append(f"fatal: {e}")
-        logger.error("Schema v3.4.3 migration failed: %s", e)
-    finally:
-        if own_connection:
-            conn.close()
-
-    return result