synapse-memory-engine 1.3.0__py3-none-any.whl

synapse_memory/core/decay_engine.py

@@ -0,0 +1,123 @@
+import math
+import time
+from typing import Any, Dict, Optional
+
+
+class DecayEngine:
+    """
+    Robust Ebbinghaus-style decay and reinforcement engine.
+
+    Formula:
+        Score = Confidence * exp(-(now - last_accessed) / Strength)
+        Strength = BaseStrength * (1 + (access_count * factor) * multiplier)
+    """
+    def __init__(
+        self,
+        category_configs: Optional[Dict[str, Dict[str, float]]] = None
+    ):
+        self.category_configs = category_configs or {}
+        self.default_config = {
+            'base_strength': 86400.0,
+            'reinforcement_factor': 0.45
+        }
+        self.decay_floor = 0.10
+
+    def _get_config(self, category: str) -> Dict[str, float]:
+        return self.category_configs.get(category, self.default_config)
+
+    def calculate_relevance(
+        self,
+        memory: Dict[str, Any],
+        now: Optional[float] = None
+    ) -> float:
+        """Calculates current relevance score for a memory node."""
+        if now is None:
+            now = time.time()
+
+        last_accessed = memory.get("last_accessed_at", memory["created_at"])
+        elapsed = max(0.0, now - last_accessed)
+
+        config = self._get_config(memory.get("category", "interaction"))
+
+        access_count = memory.get("access_count", 0)
+        feedback_multiplier = memory.get("feedback_multiplier", 1.0)
+
+        strength = config['base_strength'] * (
+            1 + (access_count * config['reinforcement_factor']) * feedback_multiplier
+        )
+
+        confidence = float(memory.get("confidence", 0.5))
+        return max(self.decay_floor, confidence * math.exp(-elapsed / max(1.0, strength)))
+
+    def temporal_multiplier(
+        self,
+        memory: Dict[str, Any],
+        now: Optional[float] = None,
+    ) -> float:
+        """Temporal-decay multiplier D(m,t) in [decay_floor, 1.0].
+
+        Separated from retrieval relevance R(m,q) so query relevance is
+        never erased: combined score S(m,q,t) = R(m,q) * D(m,t).
+        """
+        if now is None:
+            now = time.time()
+        last_accessed = memory.get("last_accessed_at", memory["created_at"])
+        elapsed = max(0.0, now - last_accessed)
+        config = self._get_config(memory.get("category", "interaction"))
+        access_count = memory.get("access_count", 0)
+        feedback_multiplier = memory.get("feedback_multiplier", 1.0)
+        strength = config['base_strength'] * (
+            1 + (access_count * config['reinforcement_factor']) * feedback_multiplier
+        )
+        return max(self.decay_floor, math.exp(-elapsed / max(1.0, strength)))
+
+    def combine_scores(self, retrieval_score: float, memory: Dict[str, Any], now: Optional[float] = None) -> float:
+        """S(m,q,t) = R(m,q) * D(m,t)."""
+        return float(retrieval_score) * self.temporal_multiplier(memory, now=now)
+
+    def calculate_retention(
+        self,
+        base_relevance: float,
+        created_epoch: float,
+        access_count: int = 0,
+        now: Optional[float] = None
+    ) -> float:
+        """
+        Calculates Ebbinghaus retention for a memory based on age and access count.
+        """
+        if now is None:
+            now = time.time()
+        elapsed = max(0.0, now - created_epoch)
+        effective_half_life = 86400.0 * 2.0 * (1.0 + 0.5 * access_count)
+        decay = math.exp(-elapsed / effective_half_life)
+        return max(self.decay_floor, base_relevance * decay)
+
+    def compute_feedback_boost(
+        self,
+        current_confidence: float,
+        feedback_type: str
+    ) -> float:
+        """Helper to adjust confidence based on feedback."""
+        boost = 0.05 if feedback_type == "positive" else -0.1
+        return max(0.1, min(1.0, current_confidence + boost))
+
+    def reinforce(
+        self,
+        memory: Dict[str, Any],
+        feedback_value: float = 1.0
+    ) -> Dict[str, Any]:
+        """
+        Applies reinforcement to a memory node.
+
+        feedback_value > 0: Positive reinforcement
+        feedback_value < 0: Negative reinforcement (penalty)
+        """
+        memory["access_count"] = memory.get("access_count", 0) + 1
+        memory["last_accessed_at"] = time.time()
+
+        current_multiplier = memory.get("feedback_multiplier", 1.0)
+        memory["feedback_multiplier"] = max(
+            0.1,
+            current_multiplier + (feedback_value * 0.1)
+        )
+        return memory

synapse_memory/core/deduplicator.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass
+from enum import Enum
+import logging
+import math
+from typing import Any, Dict, List, Optional
+
+
+logger = logging.getLogger("Deduplicator")
+
+
+class Action(Enum):
+    CREATE = "CREATE"
+    MERGE = "MERGE"
+    REJECT = "REJECT"
+
+
+@dataclass
+class MergeResult:
+    action: Action
+    target_id: Optional[str]
+    reason: str
+    provenance: Dict[str, Any]
+
+
+# Alias for backwards compatibility
+DeduplicationResult = MergeResult
+
+
+class Deduplicator:
+    def __init__(self, category_thresholds: Dict[str, float], default_threshold: float = 0.85):
+        self.category_thresholds = category_thresholds
+        self.default_threshold = default_threshold
+
+    def _get_threshold(self, category: str) -> float:
+        return self.category_thresholds.get(category, self.default_threshold)
+
+    def _jaccard_similarity(self, content_a: str, content_b: str) -> float:
+        set_a = set(content_a.lower().split())
+        set_b = set(content_b.lower().split())
+        intersection = len(set_a.intersection(set_b))
+        union = len(set_a.union(set_b))
+        return intersection / union if union > 0 else 0.0
+
+    def _cosine_similarity(self, vec_a: List[float], vec_b: List[float]) -> float:
+        if not vec_a or not vec_b or len(vec_a) != len(vec_b):
+            return 0.0
+        dot_product = sum(a * b for a, b in zip(vec_a, vec_b))
+        norm_a = math.sqrt(sum(a * a for a in vec_a))
+        norm_b = math.sqrt(sum(b * b for b in vec_b))
+        if norm_a == 0.0 or norm_b == 0.0:
+            return 0.0
+        return dot_product / (norm_a * norm_b)
+
+    def check_duplicate(self, new_mem: Dict[str, Any], existing_mems: List[Dict[str, Any]]) -> MergeResult:
+        threshold = self._get_threshold(new_mem.get("category", "interaction"))
+
+        for existing in existing_mems:
+            # 1. Exact Match Check (e.g., hash of content or specific ID)
+            if new_mem.get("id") == existing.get("id"):
+                return MergeResult(Action.REJECT, existing["id"], "Exact ID match", {"type": "exact"})
+
+            # 2. Lexical Similarity (Jaccard)
+            lexical_sim = self._jaccard_similarity(new_mem["content"], existing["content"])
+
+            # 3. Semantic Similarity (Cosine)
+            semantic_sim = self._cosine_similarity(new_mem.get("embedding", []), existing.get("embedding", []))
+
+            # 4. Conflict & Supersession Detection (Production-Grade Heuristic)
+            if semantic_sim > threshold:
+                # If high semantic similarity, compare metadata to decide supersession
+                new_conf = new_mem.get("confidence", 0.5)
+                old_conf = existing.get("confidence", 0.5)
+
+                # If new memory has significantly higher confidence, suggest supersession (merge)
+                if new_conf > old_conf + 0.1:
+                    return MergeResult(
+                        Action.MERGE,
+                        existing["id"],
+                        "Higher confidence supersession",
+                        {"new_conf": new_conf, "old_conf": old_conf}
+                    )
+
+                # Check for explicit contradiction in content
+                content_new = new_mem["content"].lower()
+                content_old = existing["content"].lower()
+                if ("true" in content_new and "false" in content_old) or ("false" in content_new and "true" in content_old):
+                    return MergeResult(Action.REJECT, None, "Direct contradiction detected", {"type": "conflict"})
+
+            # 5. Threshold Decision
+            if semantic_sim >= threshold or (lexical_sim > 0.9 and semantic_sim > 0.7):
+                return MergeResult(
+                    Action.MERGE,
+                    existing["id"],
+                    "Semantic/Lexical similarity threshold exceeded",
+                    {"semantic_sim": semantic_sim, "lexical_sim": lexical_sim}
+                )
+
+        return MergeResult(Action.CREATE, None, "No duplicates found", {})

synapse_memory/core/embedder.py

@@ -0,0 +1,119 @@
+import logging
+import os
+from typing import List, Optional
+from synapse_memory.core.embedding_provider import (
+    EmbeddingManager,
+    LocalEmbeddingProvider,
+    GeminiEmbeddingProvider,
+    OpenAIEmbeddingProvider,
+    SentenceTransformerEmbeddingProvider
+)
+
+logger = logging.getLogger("SynapseEmbedder")
+
+
+class SynapseEmbedder:
+    """
+    Multi-model embedding client. Provides seamless failover between Google Gemini API,
+    OpenAI API, SentenceTransformers, and deterministic hash vector providers.
+    """
+
+    def __init__(self, provider: Optional[str] = None, allow_fallback: Optional[bool] = None):
+        env_provider = os.getenv("SYNAPSE_EMBEDDING_PROVIDER")
+        self.provider_name = (provider or env_provider or "deterministic").lower()
+        self.model_name = os.getenv("SYNAPSE_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+        self.gemini_key = os.getenv("GEMINI_API_KEY", "")
+        self.openai_key = os.getenv("OPENAI_API_KEY", "")
+
+        is_production = os.getenv("NODE_ENV", "").lower() == "production"
+        if allow_fallback is None:
+            allow_fallback = not is_production
+
+        provider_instance = self._get_provider(allow_fallback=allow_fallback)
+        self.manager = EmbeddingManager(provider_instance)
+
+        # Validate configured dimension if explicitly set
+        configured_dim = os.getenv("SYNAPSE_EMBEDDING_DIMENSION")
+        if configured_dim:
+            expected = int(configured_dim)
+            actual = self.manager.provider.dimension
+            if actual != expected:
+                raise RuntimeError(
+                    f"Embedding dimension mismatch: provider '{self.provider_name}' produces {actual}-d vectors, "
+                    f"but SYNAPSE_EMBEDDING_DIMENSION is configured as {expected}-d."
+                )
+
+    @property
+    def dimension(self) -> int:
+        return self.manager.provider.dimension
+
+    @property
+    def is_semantic(self) -> bool:
+        return self.manager.provider.is_semantic
+
+    def _get_provider(self, allow_fallback: bool = False):
+        if self.provider_name in ("deterministic", "hash", "local"):
+            # Deterministic hash provider: always available, no external deps,
+            # but NOT a real semantic embedding model. Safe for offline/dev use;
+            # production must not silently run on it.
+            is_production = os.getenv("NODE_ENV", "").lower() == "production"
+            allow_hash = os.getenv("SYNAPSE_ALLOW_DETERMINISTIC_EMBEDDING", "false").lower() == "true"
+            if is_production and not allow_hash:
+                raise RuntimeError(
+                    "Refusing to start in production with the deterministic hash "
+                    "embedding provider (it is not a semantic model). "
+                    "Set SYNAPSE_EMBEDDING_PROVIDER to a real semantic provider "
+                    "(gemini, openai, sentence-transformers) or opt in with "
+                    "SYNAPSE_ALLOW_DETERMINISTIC_EMBEDDING=true."
+                )
+            dim = int(os.getenv("SYNAPSE_EMBEDDING_DIMENSION", "128"))
+            return LocalEmbeddingProvider(dim=dim)
+        elif self.provider_name == "gemini":
+            if not self.gemini_key:
+                if allow_fallback:
+                    logger.warning(
+                        "GEMINI_API_KEY not set; falling back to deterministic hash provider."
+                    )
+                    dim = int(os.getenv("SYNAPSE_EMBEDDING_DIMENSION", "128"))
+                    return LocalEmbeddingProvider(dim=dim)
+                raise RuntimeError(
+                    "SYNAPSE_EMBEDDING_PROVIDER=gemini but GEMINI_API_KEY is not set. "
+                    "Set GEMINI_API_KEY or switch to provider=deterministic."
+                )
+            return GeminiEmbeddingProvider(self.gemini_key)
+        elif self.provider_name == "openai":
+            if not self.openai_key:
+                if allow_fallback:
+                    logger.warning(
+                        "OPENAI_API_KEY not set; falling back to deterministic hash provider."
+                    )
+                    dim = int(os.getenv("SYNAPSE_EMBEDDING_DIMENSION", "128"))
+                    return LocalEmbeddingProvider(dim=dim)
+                raise RuntimeError(
+                    "SYNAPSE_EMBEDDING_PROVIDER=openai but OPENAI_API_KEY is not set. "
+                    "Set OPENAI_API_KEY or switch to provider=deterministic."
+                )
+            return OpenAIEmbeddingProvider(self.openai_key)
+        elif self.provider_name == "sentence-transformers":
+            # Explicit opt-in: requires [local-models] extra and model availability.
+            return SentenceTransformerEmbeddingProvider(
+                model_name=self.model_name,
+                allow_fallback=allow_fallback
+            )
+        # Unknown provider: fail closed in production, fall back in dev.
+        if allow_fallback:
+            logger.warning(
+                f"Unknown embedding provider '{self.provider_name}'; falling back to deterministic hash provider."
+            )
+            dim = int(os.getenv("SYNAPSE_EMBEDDING_DIMENSION", "128"))
+            return LocalEmbeddingProvider(dim=dim)
+        raise RuntimeError(
+            f"Unknown embedding provider '{self.provider_name}'. "
+            "Use one of: deterministic, gemini, openai, sentence-transformers."
+        )
+
+    def embed_query(self, text: str) -> List[float]:
+        """Generates a dense float vector representation for text strings."""
+        if not text.strip():
+            return [0.0] * self.manager.provider.dimension
+        return self.manager.get_embedding(text)

synapse_memory/core/embedding_provider.py

@@ -0,0 +1,241 @@
+import hashlib
+import json
+import logging
+import math
+import re
+import urllib.request
+from abc import ABC, abstractmethod
+from functools import lru_cache
+from typing import List, Optional
+
+from synapse_memory.core.observability import instrument_operation
+
+logger = logging.getLogger("EmbeddingProvider")
+
+
+class EmbeddingProvider(ABC):
+    """Base class for all embedding providers."""
+
+    @abstractmethod
+    def embed(self, text: str) -> List[float]:
+        pass
+
+    @abstractmethod
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        pass
+
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        pass
+
+    @property
+    def is_semantic(self) -> bool:
+        return True
+
+
+class DeterministicHashEmbeddingProvider(EmbeddingProvider):
+    """
+    Local deterministic hash provider for offline unit testing and development.
+    NOTE: This generates deterministic word-frequency hash projections, NOT neural semantic embeddings.
+    """
+    def __init__(self, dim: int = 128):
+        self._dim = dim
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @property
+    def is_semantic(self) -> bool:
+        return False
+
+    def embed(self, text: str) -> List[float]:
+        # Normalize text (lowercase and strip non-alphanumeric characters)
+        normalized = re.sub(r"[^\w\s]", "", text.lower()).strip()
+        words = normalized.split()
+        if not words:
+            return [0.0] * self._dim
+
+        vector = [0.0] * self._dim
+        for i, word in enumerate(words):
+            # SHA-256 digest instead of the built-in hash(): Python string hashing
+            # is salted per interpreter (PYTHONHASHSEED), so hash(word) is stable
+            # only within a single process and changes across invocations -- it must
+            # never back a stored index or a repeatable offline test.
+            digest = hashlib.sha256(word.encode("utf-8")).digest()
+            h = int.from_bytes(digest[:8], "big", signed=True)
+            idx = abs(h) % self._dim
+            sign = 1.0 if h >= 0 else -1.0
+            vector[idx] += sign * (1.0 / (1.0 + 0.05 * i))
+
+        # L2 normalization for accurate cosine similarities
+        norm = math.sqrt(sum(x * x for x in vector))
+        if norm > 0.0:
+            vector = [x / norm for x in vector]
+        return vector
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        return [self.embed(t) for t in texts]
+
+
+# Alias for backward compatibility
+LocalEmbeddingProvider = DeterministicHashEmbeddingProvider
+
+
+class SentenceTransformerEmbeddingProvider(EmbeddingProvider):
+    """
+    Production-grade local neural semantic embedding provider using sentence-transformers.
+    Requires: sentence-transformers (pip install 'synapse-memory-engine[local-models]')
+    """
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2", allow_fallback: bool = False):
+        self.model_name = model_name
+        self._model = None
+        self._fallback: Optional[DeterministicHashEmbeddingProvider] = None
+        self._is_using_real_model = False
+
+        try:
+            from sentence_transformers import SentenceTransformer
+            self._model = SentenceTransformer(model_name)
+            self._dim = int(self._model.get_sentence_embedding_dimension())
+            self._is_using_real_model = True
+            logger.info(f"Loaded neural embedding model '{model_name}' (dim: {self._dim}).")
+        except Exception as err:
+            if not allow_fallback:
+                raise ImportError(
+                    f"Failed to load semantic embedding model '{model_name}': {err}. "
+                    "For production neural embeddings, install sentence-transformers or set provider='deterministic'."
+                ) from err
+            logger.warning(
+                f"[SynapseEmbedder Warning] sentence-transformers not installed ({err}). "
+                "Falling back to DeterministicHashEmbeddingProvider for non-production development. "
+                "For production neural embeddings, install with: pip install sentence-transformers"
+            )
+            self._fallback = DeterministicHashEmbeddingProvider()
+            self._dim = self._fallback.dimension
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @property
+    def is_semantic(self) -> bool:
+        return self._is_using_real_model
+
+    @instrument_operation("st_embed")
+    def embed(self, text: str) -> List[float]:
+        if self._model is not None:
+            return self._model.encode(text).tolist()
+        if self._fallback is None:
+            self._fallback = DeterministicHashEmbeddingProvider()
+        return self._fallback.embed(text)
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        if self._model is not None:
+            return self._model.encode(texts).tolist()
+        if self._fallback is None:
+            self._fallback = DeterministicHashEmbeddingProvider()
+        return self._fallback.embed_batch(texts)
+
+
+class GeminiEmbeddingProvider(EmbeddingProvider):
+    """
+    Google Gemini text embedding provider.
+
+    NOTE: text-embedding-004 was shut down on 2026-01-14 and is retired.
+    This provider now targets gemini-embedding-001 (3072 dimensions by default),
+    the supported text-only replacement. Embeddings from the two model families
+    are incompatible -- existing text-embedding-004 vectors must be re-embedded
+    with `reembed_all_memories()` (or the equivalent migration job) before
+    querying, since changing the column definition alone does not recalculate
+    stored vectors. See `docs/production-hardening-baseline.md` and
+    `docs/backup-recovery.md` for the batched re-embed/key-rotation sweep
+    pattern.
+    """
+    def __init__(self, api_key: str, dim: int = 3072):
+        self.api_key = api_key
+        self._dim = dim
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @property
+    def is_semantic(self) -> bool:
+        return True
+
+    @instrument_operation("gemini_embed")
+    def embed(self, text: str) -> List[float]:
+        url = (
+            "https://generativelanguage.googleapis.com/v1beta/models/"
+            f"gemini-embedding-001:embedContent?key={self.api_key}"
+        )
+        headers = {"Content-Type": "application/json"}
+        payload = {"content": {"parts": [{"text": text}]}}
+
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(payload).encode("utf-8"),
+            headers=headers,
+            method="POST"
+        )
+        with urllib.request.urlopen(req, timeout=5) as response:
+            res_data = json.loads(response.read().decode("utf-8"))
+            return res_data["embedding"]["values"]
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        return [self.embed(t) for t in texts]
+
+
+class OpenAIEmbeddingProvider(EmbeddingProvider):
+    def __init__(self, api_key: str, model: str = "text-embedding-3-small", dim: int = 1536):
+        self.api_key = api_key
+        self.model = model
+        self._dim = dim
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    @property
+    def is_semantic(self) -> bool:
+        return True
+
+    @instrument_operation("openai_embed")
+    def embed(self, text: str) -> List[float]:
+        url = "https://api.openai.com/v1/embeddings"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}"
+        }
+        payload = {
+            "input": text,
+            "model": self.model
+        }
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(payload).encode("utf-8"),
+            headers=headers,
+            method="POST"
+        )
+        with urllib.request.urlopen(req, timeout=5) as response:
+            res_data = json.loads(response.read().decode("utf-8"))
+            return res_data["data"][0]["embedding"]
+
+    def embed_batch(self, texts: List[str]) -> List[List[float]]:
+        return [self.embed(t) for t in texts]
+
+
+class EmbeddingManager:
+    """
+    Manages embedding provider operations with caching and observability.
+    """
+    def __init__(self, provider: EmbeddingProvider):
+        self.provider = provider
+
+    @lru_cache(maxsize=1024)
+    def get_embedding(self, text: str) -> List[float]:
+        return self.provider.embed(text)
+
+    def get_batch_embeddings(self, texts: List[str]) -> List[List[float]]:
+        return self.provider.embed_batch(texts)