synapse-memory-engine 1.3.0__py3-none-any.whl

synapse_memory/core/memory_manager.py

@@ -0,0 +1,233 @@
+import os
+import time
+import logging
+import asyncio
+import uuid
+from typing import Dict, Any, Tuple, Optional, Union
+from synapse_memory.core.sqlite_db import SQLiteMemoryStore
+from synapse_memory.core.pgvector_db import PGVectorMemoryStore
+from synapse_memory.core.embedder import SynapseEmbedder
+from synapse_memory.core.deduplicator import Deduplicator, Action
+from synapse_memory.core.pruner import PruningEngine, CapacityPruningPolicy
+from synapse_memory.core.decay_engine import DecayEngine
+from synapse_memory.core.observability import log_event, instrument_operation
+
+logger = logging.getLogger("SynapseMemoryManager")
+
+# Explicit, operator-visible capacity ceiling for the in-memory store.
+# Override with SYNAPSE_MAX_RECORD_LIMIT (must be a positive integer).
+# The historical default was 50 records; change this only with capacity planning.
+try:
+    DEFAULT_MAX_RECORD_LIMIT = int(os.environ.get("SYNAPSE_MAX_RECORD_LIMIT", "50"))
+    if DEFAULT_MAX_RECORD_LIMIT <= 0:
+        raise ValueError
+except (TypeError, ValueError):
+    DEFAULT_MAX_RECORD_LIMIT = 50
+    logger.warning(
+        "Invalid SYNAPSE_MAX_RECORD_LIMIT; using default max_record_limit=50"
+    )
+
+
+def estimate_token_cost(content: str) -> int:
+    """Production token estimator: prefers tiktoken when installed, else safe heuristic.
+    Heuristic calibrated to ~4 chars/token (OpenAI/Gemini average for English).
+    """
+    try:
+        import tiktoken  # type: ignore
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        return max(15, len(enc.encode(content)) + 8)
+    except Exception:
+        # Fallback: chars/4 + overhead. More accurate than words*1.35 for code/CJK.
+        return max(15, int(len(content) / 4) + 10)
+
+
+class MemoryManager:
+    """
+    Manages high-level memory operations: Semantic Deduplication,
+    Multi-tenant Ingestion, Auto-Eviction, and Cognitive Memory Pruning.
+    Compatible with both SQLiteMemoryStore and PGVectorMemoryStore backends.
+
+    Capacity: ``max_record_limit`` bounds how many memories a tenant may hold.
+    After every ingest, :meth:`auto_prune_store` evicts the lowest-relevance
+    memories until the count is within the limit. The default comes from the
+    ``SYNAPSE_MAX_RECORD_LIMIT`` environment variable and falls back to 50
+    records (the historical default). Expose this setting explicitly -- never
+    rely on the silent default in production, or older memories are pruned
+    without the operator realizing a cap is active.
+    """
+
+    def __init__(
+        self,
+        store: Union[SQLiteMemoryStore, PGVectorMemoryStore, Any],
+        embedder: SynapseEmbedder,
+        category_thresholds: Optional[Dict[str, float]] = None,
+        max_record_limit: int = DEFAULT_MAX_RECORD_LIMIT
+    ):
+        self.store = store
+        self.embedder = embedder
+        self.deduplicator = Deduplicator(category_thresholds or {})
+        self.max_record_limit = int(max_record_limit)
+        self.decay_engine = DecayEngine({})
+        self.pruner = PruningEngine([CapacityPruningPolicy(max_record_limit, self.decay_engine)])
+
+    @instrument_operation("ingest_memory")
+    def ingest_with_deduplication(
+        self,
+        content: str,
+        category: str = "interaction",
+        confidence: float = 0.95,
+        tenant_id: str = "default"
+    ) -> Tuple[str, str, int]:
+        """
+        Ingests a new memory node.
+        Runs semantic similarity comparison against existing indexed nodes within the same tenant.
+
+        When a near-duplicate is found (Action.MERGE), the incoming content is preserved:
+        a new memory row is created and linked to the old one via a supersession chain
+        (``superseded_by``). The old row is marked superseded but retained for provenance.
+        Confidence is merged conservatively: the higher of the two confidences is kept,
+        and a small reinforcement is applied only when the new content is at least as
+        confident as the existing one. This ensures updated/corrected facts are never
+        silently discarded.
+
+        When Action.REJECT is returned (direct contradiction), the incoming memory is
+        still recorded as a separate row so no information is lost; the caller can
+        decide how to resolve the conflict.
+        """
+        # 1. Generate dense vector representation
+        new_vector = self.embedder.embed_query(content)
+        existing_memories = self.store.get_memories_by_category(category, tenant_id=tenant_id)
+
+        # 2. Check for duplicate nodes
+        new_node_base = {
+            "content": content,
+            "category": category,
+            "tenant_id": tenant_id,
+            "embedding": new_vector,
+            "confidence": confidence,
+        }
+        result = self.deduplicator.check_duplicate(new_node_base, existing_memories)
+
+        # 3. Handle Deduplication Result
+        if result.action == Action.MERGE and result.target_id:
+            match_id = result.target_id
+            match_mem = next((m for m in existing_memories if m["id"] == match_id), None)
+            old_confidence = float(match_mem.get("confidence", 0.5)) if match_mem else 0.5
+
+            merged_confidence = max(old_confidence, confidence)
+            if confidence >= old_confidence:
+                merged_confidence = min(1.0, merged_confidence + 0.02)
+
+            new_memory_id = f"mem_{uuid.uuid4().hex[:16]}"
+            token_cost = estimate_token_cost(content)
+            new_node = {
+                "id": new_memory_id,
+                "tenant_id": tenant_id,
+                "content": content,
+                "category": category,
+                "confidence": merged_confidence,
+                "created_at": time.time(),
+                "access_count": 0,
+                "token_cost": token_cost,
+                "embedding": new_vector,
+                "embedding_dimension": len(new_vector),
+                "model_version": getattr(self.embedder, "model_name", "unknown"),
+                "supersedes": match_id,
+                "superseded_by": None,
+            }
+            self.store.insert_memory(new_node)
+
+            try:
+                self.store.mark_superseded(match_id, new_memory_id, tenant_id=tenant_id)
+            except Exception:
+                log_event("memory_deduplication", "MERGE_MARK_FAILED", {"target_id": match_id, "tenant_id": tenant_id})
+
+            # The supersession row above is a real row: it counts toward
+            # max_record_limit like any other write, so enforce the capacity
+            # ceiling on this path too. Without this, repeated merges of the same
+            # fact grow the table without bound (the superseded lineage rows are
+            # never evicted) even though CREATE/REJECT do prune.
+            self.auto_prune_store(tenant_id=tenant_id)
+
+            log_event("memory_deduplication", "MERGED_PRESERVED", {
+                "old_id": match_id, "new_id": new_memory_id, "tenant_id": tenant_id,
+                "old_confidence": old_confidence, "new_confidence": merged_confidence,
+            })
+            return new_memory_id, "MERGED", token_cost
+
+        elif result.action == Action.REJECT:
+            log_event("memory_deduplication", "REJECTED_RECORDED", {"tenant_id": tenant_id, "reason": result.reason})
+            new_memory_id = f"mem_{uuid.uuid4().hex[:16]}"
+            token_cost = estimate_token_cost(content)
+            new_node = {
+                "id": new_memory_id,
+                "tenant_id": tenant_id,
+                "content": content,
+                "category": category,
+                "confidence": confidence,
+                "created_at": time.time(),
+                "access_count": 0,
+                "token_cost": token_cost,
+                "embedding": new_vector,
+                "embedding_dimension": len(new_vector),
+                "model_version": getattr(self.embedder, "model_name", "unknown"),
+                "supersedes": None,
+                "superseded_by": None,
+            }
+            self.store.insert_memory(new_node)
+            self.auto_prune_store(tenant_id=tenant_id)
+            return new_memory_id, "REJECTED", token_cost
+
+        # 4. Otherwise, save as a clean memory
+        log_event("memory_deduplication", "CREATED", {"tenant_id": tenant_id})
+        memory_id = f"mem_{uuid.uuid4().hex[:16]}"
+        token_cost = estimate_token_cost(content)
+        new_node = {
+            "id": memory_id,
+            "tenant_id": tenant_id,
+            "content": content,
+            "category": category,
+            "confidence": confidence,
+            "created_at": time.time(),
+            "access_count": 0,
+            "token_cost": token_cost,
+            "embedding": new_vector,
+            # Embedding provenance: stamps which dimension + model wrote this row,
+            # so schema/drift checks can detect stale index entries after an
+            # embedding-model upgrade (and trigger a re-embed pass).
+            "embedding_dimension": len(new_vector),
+            "model_version": getattr(self.embedder, "model_name", "unknown"),
+            "supersedes": None,
+            "superseded_by": None,
+        }
+        self.store.insert_memory(new_node)
+
+        # 5. Execute Auto-Pruning if limit is breached
+        self.auto_prune_store(tenant_id=tenant_id)
+
+        return memory_id, "CREATED", token_cost
+
+    async def ingest_async(self, *args, **kwargs) -> Tuple[str, str, int]:
+        return await asyncio.to_thread(self.ingest_with_deduplication, *args, **kwargs)
+
+    def auto_prune_store(self, tenant_id: Optional[str] = None) -> int:
+        """
+        Evicts memories based on the configured PruningEngine within a tenant.
+
+        Returns:
+            The number of successfully evicted memory nodes.
+        """
+        memories = self.store.get_all_memories(tenant_id=tenant_id, include_superseded=True)
+        targets = self.pruner.run_pruning(memories)
+
+        evicted = 0
+        for target_id in targets:
+            self.store.delete_memory(target_id, tenant_id=tenant_id)
+            evicted += 1
+            log_event("memory_pruning", "EVICTED", {"target_id": target_id, "tenant_id": tenant_id})
+
+        return evicted
+
+    async def prune_async(self, tenant_id: Optional[str] = None) -> int:
+        return await asyncio.to_thread(self.auto_prune_store, tenant_id=tenant_id)

synapse_memory/core/migrations.py

@@ -0,0 +1,241 @@
+"""
+Lightweight, dependency-free forward-only schema migrations for SynapseMemory.
+
+Why not Alembic: the engine must stay importable and runnable with zero external
+services (SQLite) and a minimal dependency footprint. This runner is versioned,
+idempotent, and records applied versions in a `schema_migrations` ledger table so
+a team can audit exactly which schema state a database is in. It supports both
+SQLite (local/dev) and PostgreSQL (team/prod) via a tiny dialect shim.
+
+Adding a migration:
+  1. Append a Migration(version=N, name=..., sqlite=..., postgres=...) to MIGRATIONS.
+  2. Bump CURRENT_SCHEMA_VERSION to N.
+Never edit an already-released migration — add a new one.
+"""
+
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any, Callable, List, Optional
+
+logger = logging.getLogger("SynapseMigrations")
+
+# Ledger + metadata columns tracked on the memories table so embedding-model
+# upgrades (e.g. 128-d hash -> 384-d MiniLM) are explicit, not silent.
+CURRENT_SCHEMA_VERSION = 5
+
+
+@dataclass
+class Migration:
+    version: int
+    name: str
+    # Callables receive a cursor + a `p` placeholder ('?' for sqlite, '%s' for pg)
+    sqlite: Optional[Callable[[Any], None]] = None
+    postgres: Optional[Callable[[Any], None]] = None
+
+
+def _column_exists(cur: Any, table: str, column: str) -> bool:
+    """Dialect-aware column existence probe (PostgreSQL then SQLite)."""
+    try:
+        cur.execute(
+            "SELECT 1 FROM information_schema.columns WHERE table_name=%s AND column_name=%s",
+            (table, column),
+        )
+        if cur.fetchone():
+            return True
+        return False
+    except Exception:
+        # SQLite has no information_schema; fall through to PRAGMA.
+        pass
+    try:
+        cur.execute(f"PRAGMA table_info({table});")
+        for row in cur.fetchall():
+            if isinstance(row, dict):
+                if row.get("name") == column:
+                    return True
+            elif len(row) > 1 and row[1] == column:
+                return True
+        return False
+    except Exception:
+        return False
+
+
+def _add_column_if_missing(cur: Any, table: str, column: str, coltype: str) -> None:
+    """Idempotent ADD COLUMN that works on both SQLite and PostgreSQL.
+
+    Only a real duplicate-column condition counts as success. Any other failure
+    — missing privileges, invalid DDL, lock timeout, schema drift — is re-raised
+    so ``fail_closed=True`` aborts startup instead of recording a migration
+    version for a column that was never created (F-12).
+    """
+    if _column_exists(cur, table, column):
+        return
+    try:
+        cur.execute(f"ALTER TABLE {table} ADD COLUMN {column} {coltype};")
+    except Exception:
+        # Re-probe: a concurrent replica may have added it, in which case this
+        # is genuinely done. Otherwise the failure is real and must propagate.
+        if _column_exists(cur, table, column):
+            return
+        raise
+
+
+def _m1_embedding_metadata(cur: Any) -> None:
+    """v1: Track embedding dimension + model version per memory row."""
+    _add_column_if_missing(cur, "memories", "embedding_dimension", "INTEGER NOT NULL DEFAULT 0")
+    _add_column_if_missing(cur, "memories", "model_version", "VARCHAR(64) NOT NULL DEFAULT 'unknown'")
+
+
+def _m2_migration_ledger(cur: Any) -> None:
+    """v2: Audit ledger of applied schema versions (created here, not via ADD COLUMN)."""
+    cur.execute(
+        """
+        CREATE TABLE IF NOT EXISTS schema_migrations (
+            version INTEGER PRIMARY KEY,
+            name VARCHAR(128) NOT NULL,
+            applied_at DOUBLE PRECISION NOT NULL
+        );
+        """
+    )
+
+
+def _m3_backfill_dimension(cur: Any) -> None:
+    """v3: Backfill embedding_dimension for rows written before tracking existed."""
+    # Rows with dimension 0 keep 0 (unknown); a later re-embed pass fills them.
+    cur.execute("UPDATE memories SET embedding_dimension = 0 WHERE embedding_dimension IS NULL;")
+
+
+def _m4_content_plain_search_projection(cur: Any) -> None:
+    """v4: Combined-protection search projection for field-encrypted content.
+
+    Adds `content_plain` (searchable projection feeding the tsvector index).
+    Existing rows backfill `content_plain` from `content` ONLY when that
+    column still holds plaintext — i.e. rows written before P0-2 field
+    encryption. Rows already holding Fernet ciphertext are left untouched
+    (re-encrypting ciphertext would destroy recoverability); operators must
+    run the documented key-rotation/re-encryption sweep for those.
+    Detection heuristic: Fernet tokens are urlsafe-base64 starting with
+    'gAAAAA'; anything else is treated as legacy plaintext.
+    """
+    _add_column_if_missing(cur, "memories", "content_plain", "TEXT NOT NULL DEFAULT ''")
+    cur.execute(
+        "UPDATE memories SET content_plain = content "
+        "WHERE (content_plain IS NULL OR content_plain = '') "
+        "AND content NOT LIKE 'gAAAAA%';"
+    )
+
+
+def _m5_supersession_chain(cur: Any) -> None:
+    """v5: Deduplication supersession lineage (MERGE keeps both rows).
+
+    `supersedes` points from the successor (new) row to the row it replaced;
+    `superseded_by` points from the replaced (old) row to its successor.
+    """
+    _add_column_if_missing(cur, "memories", "supersedes", "VARCHAR(128)")
+    _add_column_if_missing(cur, "memories", "superseded_by", "VARCHAR(128)")
+
+
+MIGRATIONS: List[Migration] = [
+    Migration(1, "embedding_metadata", _m1_embedding_metadata, _m1_embedding_metadata),
+    Migration(2, "migration_ledger", _m2_migration_ledger, _m2_migration_ledger),
+    Migration(3, "backfill_dimension", _m3_backfill_dimension, _m3_backfill_dimension),
+    Migration(4, "content_plain_search_projection", _m4_content_plain_search_projection, _m4_content_plain_search_projection),
+    Migration(5, "supersession_chain", _m5_supersession_chain, _m5_supersession_chain),
+]
+
+
+def _applied_versions(cur: Any) -> set:
+    try:
+        cur.execute("SELECT version FROM schema_migrations;")
+        rows = cur.fetchall()
+        return {int(r[0] if isinstance(r, (tuple, list)) else r["version"]) for r in rows}
+    except Exception:
+        # Ledger table doesn't exist yet -> nothing applied.
+        return set()
+
+
+def run_migrations(
+    get_connection: Callable[[], Any],
+    is_postgres: bool,
+    fail_closed: bool = False,
+    close_on_exit: bool = True,
+    owns_connection: bool = True,
+) -> int:
+    """Apply all pending forward migrations. Returns count applied. Idempotent.
+
+    Args:
+        get_connection: Callable returning a fresh connection for this run.
+        is_postgres: Dialect to apply (SQLite vs PostgreSQL).
+        fail_closed: Raise on migration failure instead of skipping.
+        close_on_exit: If True (default), the connection is released when the run
+            finishes. Set False only when the caller keeps the connection alive
+            and reuses it afterwards, e.g. SQLiteMemoryStore caching its
+            schema-init ':memory:' connection.
+        owns_connection: If True (default), treat the returned connection as
+            freshly-owned (non-pooled) and close it in the finally block. Set
+            False when the connection belongs to an external pool that must be
+            returned, not closed. Existing callers can rely on the default True
+            (fresh connection). PGVectorMemoryStore passes a pooled connection
+            and sets this False.
+
+        The connection is closed only when ``owns_connection`` AND
+        ``close_on_exit`` are both True. Otherwise it is handed back through the
+        factory owner's ``_return_connection`` hook (when present) or closed as
+        a last-resort fallback.
+    """
+    applied_total = 0
+    conn = get_connection()
+    should_close = owns_connection and close_on_exit
+    try:
+        cur = conn.cursor()
+        # The ledger (v2) must exist before we can read it; bootstrap directly.
+        _m2_migration_ledger(cur)
+        conn.commit()
+
+        done = _applied_versions(cur)
+        for migration in MIGRATIONS:
+            if migration.version in done:
+                continue
+            fn = migration.postgres if is_postgres else migration.sqlite
+            if fn is None:
+                continue
+            try:
+                fn(cur)
+                cur.execute(
+                    "INSERT INTO schema_migrations (version, name, applied_at) VALUES (%s, %s, %s);"
+                    if is_postgres
+                    else "INSERT INTO schema_migrations (version, name, applied_at) VALUES (?, ?, ?);",
+                    (migration.version, migration.name, time.time()),
+                )
+                conn.commit()
+                applied_total += 1
+                logger.info("Applied migration v%d: %s", migration.version, migration.name)
+            except Exception as e:
+                conn.rollback()
+                if fail_closed:
+                    raise RuntimeError(f"Migration v{migration.version} failed: {e}") from e
+                logger.warning("Migration v%d (%s) skipped: %s", migration.version, migration.name, e)
+    finally:
+        try:
+            if should_close:
+                conn.close()
+            else:
+                # Return pooled connections to the pool instead of closing.
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+                try:
+                    owner = getattr(get_connection, "__self__", None)
+                    if owner is not None:
+                        owner._return_connection(conn)
+                    else:
+                        conn.close()
+                except Exception:
+                    try:
+                        conn.close()
+                    except Exception:
+                        pass
+        except Exception:
+            pass
+    return applied_total

synapse_memory/core/observability.py

@@ -0,0 +1,48 @@
+import logging
+import json
+import time
+from typing import Dict, Any
+from functools import wraps
+
+# Structured Logger Setup
+logger = logging.getLogger("SynapseObservability")
+
+
+def log_event(event_name: str, status: str, details: Dict[str, Any], level: int = logging.INFO):
+    """Logs structured JSON events for production analysis."""
+    log_record = {
+        "event": event_name,
+        "status": status,
+        "timestamp": time.time(),
+        "details": details
+    }
+    logger.log(level, json.dumps(log_record))
+
+
+def instrument_operation(operation_name: str):
+    """Decorator to instrument method execution times and failures."""
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            start = time.perf_counter()
+            try:
+                result = func(*args, **kwargs)
+                log_event(operation_name, "SUCCESS", {"duration_ms": (time.perf_counter() - start) * 1000})
+                return result
+            except Exception as e:
+                log_event(operation_name, "FAILURE", {"error": str(e), "duration_ms": (time.perf_counter() - start) * 1000}, level=logging.ERROR)
+                raise e
+        return wrapper
+    return decorator
+
+
+class MetricsCollector:
+    """Simple in-memory metrics collector."""
+    def __init__(self):
+        self.metrics: Dict[str, float] = {}
+
+    def increment(self, metric_name: str, value: float = 1.0):
+        self.metrics[metric_name] = self.metrics.get(metric_name, 0.0) + value
+
+    def get_metrics(self) -> Dict[str, float]:
+        return self.metrics