superlocalmemory 2.8.5 → 3.0.0

package/src/superlocalmemory/core/config.py

@@ -0,0 +1,391 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Configuration.
+
+Unified configuration with Mode A/B/C capability matrix.
+Clean — zero dead options, every config has a consumer.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from superlocalmemory.storage.models import Mode
+
+
+# ---------------------------------------------------------------------------
+# Default Paths
+# ---------------------------------------------------------------------------
+
+DEFAULT_BASE_DIR = Path.home() / ".superlocalmemory"
+DEFAULT_DB_NAME = "memory.db"
+DEFAULT_PROFILES_FILE = "profiles.json"
+
+
+# ---------------------------------------------------------------------------
+# Embedding Config
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class EmbeddingConfig:
+    """Embedding model configuration per mode."""
+
+    model_name: str = "nomic-ai/nomic-embed-text-v1.5"
+    dimension: int = 768
+    # Azure / cloud settings (Mode C only)
+    api_endpoint: str = ""
+    api_key: str = ""
+    api_version: str = "2024-02-01"
+    deployment_name: str = ""
+
+    @property
+    def is_cloud(self) -> bool:
+        return bool(self.api_endpoint)
+
+
+# ---------------------------------------------------------------------------
+# LLM Config
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class LLMConfig:
+    """LLM provider configuration per mode."""
+
+    provider: str = ""             # "" = no LLM, "ollama", "azure", "openai", "anthropic"
+    model: str = ""                # Model name/deployment
+    api_key: str = ""
+    api_base: str = ""
+    temperature: float = 0.0       # Deterministic by default
+    max_tokens: int = 4096
+    timeout_seconds: float = 60.0
+
+    @property
+    def is_available(self) -> bool:
+        return bool(self.provider)
+
+
+# ---------------------------------------------------------------------------
+# Channel Weights
+# ---------------------------------------------------------------------------
+
+@dataclass
+class ChannelWeights:
+    """Retrieval channel weights — 4 channels, query-adaptive."""
+
+    # Entity-linked facts are high-precision matches that rank above BM25.
+    semantic: float = 1.2
+    bm25: float = 1.0
+    entity_graph: float = 1.3
+    temporal: float = 1.0
+
+    def as_dict(self) -> dict[str, float]:
+        return {
+            "semantic": self.semantic,
+            "bm25": self.bm25,
+            "entity_graph": self.entity_graph,
+            "temporal": self.temporal,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Encoding Config
+# ---------------------------------------------------------------------------
+
+@dataclass
+class EncodingConfig:
+    """Configuration for the encoding (memory creation) pipeline."""
+
+    # Fact extraction
+    chunk_size: int = 10           # Conversation turns per extraction chunk
+    max_facts_per_chunk: int = 5   # Max facts extracted per chunk
+    min_fact_confidence: float = 0.3
+
+    # Entity resolution
+    entity_similarity_threshold: float = 0.85
+    max_entity_candidates: int = 10
+
+    # Graph construction
+    semantic_edge_top_k: int = 5   # Top-K semantic edges per new fact
+    temporal_edge_window_hours: int = 168  # 1 week
+
+    # Consolidation
+    consolidation_similarity_threshold: float = 0.85
+    max_consolidation_candidates: int = 5
+
+    # Entropy gate
+    entropy_threshold: float = 0.95
+
+
+# ---------------------------------------------------------------------------
+# Retrieval Config
+# ---------------------------------------------------------------------------
+
+@dataclass
+class RetrievalConfig:
+    """Configuration for the retrieval (recall) pipeline."""
+
+    # Fusion
+    rrf_k: int = 60               # RRF smoothing constant (D116: k=60 for diversity)
+    top_k: int = 20               # Final results to return
+
+    # Per-channel
+    semantic_top_k: int = 50      # ANN pre-filter candidates
+    bm25_top_k: int = 50
+    entity_graph_max_hops: int = 3
+    temporal_proximity_days: int = 30
+
+    # Reranking
+    use_cross_encoder: bool = True
+    cross_encoder_model: str = "BAAI/bge-reranker-v2-m3"
+
+    # Agentic (Mode C only)
+    agentic_max_rounds: int = 3
+    agentic_confidence_threshold: float = 0.3
+
+    # Spreading activation
+    spreading_activation_decay: float = 0.7
+    spreading_activation_threshold: float = 0.1
+
+    # Trust weighting — apply Bayesian trust scores to retrieval ranking.
+    # When enabled, each fact's score is multiplied by a trust weight in [0.5, 1.5].
+    # Low-trust facts are demoted; high-trust facts are promoted.
+    # Default trust = 1.0 (no effect when no trust data exists).
+    use_trust_weighting: bool = True
+
+    # Ablation channel control for experiments.
+    # List of channel names to SKIP during retrieval (e.g., ["bm25", "entity_graph"]).
+    # Used by s19_runner for ablation experiments. Empty = all channels active.
+    disabled_channels: list[str] = field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Math Config
+# ---------------------------------------------------------------------------
+
+@dataclass
+class MathConfig:
+    """Configuration for mathematical layers."""
+
+    # Fisher-Rao
+    fisher_temperature: float = 15.0
+    fisher_bayesian_update: bool = True
+    # "simplified" = local Mahalanobis-like (fast, existing behaviour)
+    # "full"       = Atkinson-Mitchell geodesic from FisherRaoMetric class
+    fisher_mode: str = "simplified"
+
+    # Langevin
+    langevin_dt: float = 0.005
+    langevin_temperature: float = 0.3
+    langevin_persist_positions: bool = True
+    langevin_weight_range: tuple[float, float] = (0.0, 1.0)
+
+    # Hopfield
+
+    # Sheaf (at encoding time, NOT retrieval)
+    sheaf_at_encoding: bool = True
+    sheaf_contradiction_threshold: float = 0.45
+    # Max edges to check per fact during sheaf consistency.
+    # At 18K+ edges, coboundary computation becomes O(N*dim^2) and hangs.
+    # Facts with more edges than this skip sheaf check (still get contradiction
+    # detection via consolidator UPDATE/SUPERSEDE path).
+    sheaf_max_edges_per_check: int = 200
+
+    # Rate-Distortion (production only, disabled for benchmarks)
+
+
+# ---------------------------------------------------------------------------
+# Master Config
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SLMConfig:
+    """Master configuration for SuperLocalMemory V3.
+
+    Create via SLMConfig.for_mode(Mode.A) for mode-specific defaults.
+    """
+
+    mode: Mode = Mode.A
+    base_dir: Path = DEFAULT_BASE_DIR
+    db_path: Path | None = None    # Computed from base_dir if None
+    active_profile: str = "default"
+
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    llm: LLMConfig = field(default_factory=LLMConfig)
+    channel_weights: ChannelWeights = field(default_factory=ChannelWeights)
+    encoding: EncodingConfig = field(default_factory=EncodingConfig)
+    retrieval: RetrievalConfig = field(default_factory=RetrievalConfig)
+    math: MathConfig = field(default_factory=MathConfig)
+
+    def __post_init__(self) -> None:
+        if self.db_path is None:
+            self.db_path = self.base_dir / DEFAULT_DB_NAME
+
+    @classmethod
+    def load(cls, config_path: Path | None = None) -> SLMConfig:
+        """Load config from JSON file. Returns default Mode A if file doesn't exist."""
+        path = config_path or (DEFAULT_BASE_DIR / "config.json")
+        if not path.exists():
+            return cls.for_mode(Mode.A)
+        import json
+        data = json.loads(path.read_text())
+        mode = Mode(data.get("mode", "a"))
+        llm_data = data.get("llm", {})
+        config = cls.for_mode(
+            mode,
+            llm_provider=llm_data.get("provider", ""),
+            llm_model=llm_data.get("model", ""),
+            llm_api_key=llm_data.get("api_key", ""),
+            llm_api_base=llm_data.get("base_url", ""),
+            embedding_endpoint=data.get("embedding", {}).get("api_endpoint", ""),
+            embedding_key=data.get("embedding", {}).get("api_key", ""),
+            embedding_deployment=data.get("embedding", {}).get("deployment_name", ""),
+        )
+        config.active_profile = data.get("active_profile", "default")
+        return config
+
+    def save(self, config_path: Path | None = None) -> None:
+        """Save config to JSON file."""
+        import json
+        path = config_path or (self.base_dir / "config.json")
+        path.parent.mkdir(parents=True, exist_ok=True)
+        data = {
+            "mode": self.mode.value,
+            "active_profile": self.active_profile,
+            "llm": {
+                "provider": self.llm.provider,
+                "model": self.llm.model,
+                "api_key": self.llm.api_key,
+                "base_url": self.llm.api_base,
+            },
+            "embedding": {
+                "model_name": self.embedding.model_name,
+                "dimension": self.embedding.dimension,
+                "api_endpoint": self.embedding.api_endpoint,
+                "api_key": self.embedding.api_key,
+                "deployment_name": self.embedding.deployment_name,
+            },
+        }
+        path.write_text(json.dumps(data, indent=2))
+
+    @staticmethod
+    def provider_presets() -> dict[str, dict[str, str]]:
+        """Provider presets for setup wizard."""
+        return {
+            "openai": {
+                "base_url": "https://api.openai.com/v1",
+                "model": "gpt-4.1-mini",
+                "embedding_model": "text-embedding-3-large",
+                "env_key": "OPENAI_API_KEY",
+            },
+            "anthropic": {
+                "base_url": "https://api.anthropic.com",
+                "model": "claude-sonnet-4-6",
+                "embedding_model": "",
+                "env_key": "ANTHROPIC_API_KEY",
+            },
+            "ollama": {
+                "base_url": "http://localhost:11434",
+                "model": "llama3.2",
+                "embedding_model": "nomic-embed-text",
+                "env_key": "",
+            },
+            "openrouter": {
+                "base_url": "https://openrouter.ai/api/v1",
+                "model": "openai/gpt-4.1-mini",
+                "embedding_model": "",
+                "env_key": "OPENROUTER_API_KEY",
+            },
+        }
+
+    @classmethod
+    def default(cls) -> SLMConfig:
+        """Create default Mode A configuration."""
+        return cls.for_mode(Mode.A)
+
+    @classmethod
+    def for_mode(
+        cls,
+        mode: Mode,
+        base_dir: Path | None = None,
+        *,
+        llm_provider: str = "",
+        llm_model: str = "",
+        llm_api_key: str = "",
+        llm_api_base: str = "",
+        embedding_endpoint: str = "",
+        embedding_key: str = "",
+        embedding_deployment: str = "",
+    ) -> SLMConfig:
+        """Create config with mode-appropriate defaults."""
+        _base = base_dir or DEFAULT_BASE_DIR
+
+        if mode == Mode.A:
+            return cls(
+                mode=mode,
+                base_dir=_base,
+                embedding=EmbeddingConfig(
+                    model_name="nomic-ai/nomic-embed-text-v1.5",
+                    dimension=768,
+                ),
+                llm=LLMConfig(),  # No LLM
+                retrieval=RetrievalConfig(
+                    use_cross_encoder=True,  # Hotpatch: CE ON for Mode A with bge-reranker
+                ),
+                math=MathConfig(
+                    sheaf_contradiction_threshold=0.45,  # 768d threshold
+                ),
+            )
+
+        if mode == Mode.B:
+            return cls(
+                mode=mode,
+                base_dir=_base,
+                embedding=EmbeddingConfig(
+                    model_name="nomic-ai/nomic-embed-text-v1.5",
+                    dimension=768,
+                ),
+                llm=LLMConfig(
+                    provider=llm_provider or "ollama",
+                    model=llm_model or "phi3:mini",
+                    api_base=llm_api_base or "http://localhost:11434",
+                ),
+                retrieval=RetrievalConfig(use_cross_encoder=True),
+            )
+
+        # Mode C — FULL POWER, UNRESTRICTED
+        return cls(
+            mode=mode,
+            base_dir=_base,
+            embedding=EmbeddingConfig(
+                model_name="text-embedding-3-large",
+                dimension=3072,
+                api_endpoint=embedding_endpoint,
+                api_key=embedding_key,
+                deployment_name=embedding_deployment,
+            ),
+            llm=LLMConfig(
+                provider=llm_provider or "azure",
+                model=llm_model or "gpt-4.1-mini",
+                api_key=llm_api_key,
+                api_base=llm_api_base,
+            ),
+            channel_weights=ChannelWeights(
+                semantic=1.5,
+                bm25=1.2,
+                entity_graph=1.3,
+                temporal=1.0,
+            ),
+            retrieval=RetrievalConfig(
+                use_cross_encoder=True,
+                semantic_top_k=80,
+                agentic_max_rounds=2,  # EverMemOS 2-round
+            ),
+            math=MathConfig(
+                sheaf_contradiction_threshold=0.65,  # Higher for 3072d embeddings
+            ),
+        )

package/src/superlocalmemory/core/embeddings.py

@@ -0,0 +1,293 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory V3 — Embedding Service.
+
+Thread-safe, dimension-validated embedding with Fisher variance computation.
+Supports local (768-dim nomic) and cloud (3072-dim) models with EXPLICIT errors
+on dimension mismatch — NEVER silently falls back to a different dimension.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+from superlocalmemory.core.config import EmbeddingConfig
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Fisher variance constants
+# ---------------------------------------------------------------------------
+_FISHER_VAR_MIN = 0.05
+_FISHER_VAR_MAX = 2.0
+_FISHER_VAR_RANGE = _FISHER_VAR_MAX - _FISHER_VAR_MIN  # 1.95
+
+
+class DimensionMismatchError(RuntimeError):
+    """Raised when the actual embedding dimension differs from config.
+
+    This is a HARD failure — V1 silently fell back to local embeddings
+    when Azure failed, changing dimension from 3072 to 768 mid-run.
+    We crash loudly instead.
+    """
+
+
+class EmbeddingService:
+    """Thread-safe embedding service with strict dimension validation.
+
+    Lazy-loads the underlying model on first embed call.
+    Validates every output dimension against the configured expectation.
+    """
+
+    def __init__(self, config: EmbeddingConfig) -> None:
+        self._config = config
+        self._model: object | None = None
+        self._lock = threading.Lock()
+        self._loaded = False
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    @property
+    def dimension(self) -> int:
+        """Expected embedding dimension (from config)."""
+        return self._config.dimension
+
+    def embed(self, text: str) -> list[float]:
+        """Embed a single text string.
+
+        Returns:
+            L2-normalized embedding of exactly ``self.dimension`` floats.
+
+        Raises:
+            ValueError: If text is empty.
+            DimensionMismatchError: If output dimension != config.
+        """
+        if not text or not text.strip():
+            raise ValueError("Cannot embed empty text")
+        vec = self._encode_single(text)
+        self._validate_dimension(vec)
+        return vec.tolist()
+
+    def embed_batch(self, texts: list[str]) -> list[list[float]]:
+        """Embed a batch of texts.
+
+        Returns:
+            List of L2-normalized embeddings, each ``self.dimension`` floats.
+
+        Raises:
+            ValueError: If any text is empty or list is empty.
+            DimensionMismatchError: If any output dimension != config.
+        """
+        if not texts:
+            raise ValueError("Cannot embed empty batch")
+        for i, t in enumerate(texts):
+            if not t or not t.strip():
+                raise ValueError(f"Text at index {i} is empty")
+
+        vectors = self._encode_batch(texts)
+        for vec in vectors:
+            self._validate_dimension(vec)
+        return [v.tolist() for v in vectors]
+
+    def compute_fisher_params(
+        self,
+        embedding: list[float],
+    ) -> tuple[list[float], list[float]]:
+        """Compute Fisher-Rao parameters from a raw embedding.
+
+        Variance is content-derived (NOT uniform). Dimensions with strong
+        signal (high absolute value) get LOW variance (high confidence).
+        Weak-signal dimensions get HIGH variance (uncertainty).
+
+        This heterogeneous variance is what gives Fisher-Rao metric
+        discriminative power beyond simple cosine similarity.
+
+        Args:
+            embedding: Raw embedding vector (already L2-normalized).
+
+        Returns:
+            (mean, variance) — both lists of ``self.dimension`` floats.
+            Variance values are clamped to [0.3, 2.0].
+        """
+        arr = np.asarray(embedding, dtype=np.float64)
+        norm = float(np.linalg.norm(arr))
+
+        if norm < 1e-10:
+            mean = np.zeros(len(arr), dtype=np.float64)
+            variance = np.full(len(arr), _FISHER_VAR_MAX, dtype=np.float64)
+            return mean.tolist(), variance.tolist()
+
+        mean = arr / norm
+
+        # Content-derived heterogeneous variance
+        abs_mean = np.abs(mean)
+        max_val = float(np.max(abs_mean)) + 1e-10
+        signal_strength = abs_mean / max_val  # [0, 1]
+
+        # Inverse: strong signal -> low variance, weak -> high
+        variance = _FISHER_VAR_MAX - _FISHER_VAR_RANGE * signal_strength
+        variance = np.clip(variance, _FISHER_VAR_MIN, _FISHER_VAR_MAX)
+
+        return mean.tolist(), variance.tolist()
+
+    # ------------------------------------------------------------------
+    # Internals — model loading
+    # ------------------------------------------------------------------
+
+    def _ensure_loaded(self) -> None:
+        """Lazy-load the model on first use (thread-safe)."""
+        if self._loaded:
+            return
+        with self._lock:
+            if self._loaded:
+                return
+            if self._config.is_cloud:
+                # Cloud mode: no local model needed, validate config
+                if not self._config.api_endpoint or not self._config.api_key:
+                    raise RuntimeError(
+                        "Cloud embedding requires api_endpoint and api_key"
+                    )
+                logger.info(
+                    "EmbeddingService: cloud mode (%s, %d-dim)",
+                    self._config.deployment_name,
+                    self._config.dimension,
+                )
+            else:
+                self._load_local_model()
+            self._loaded = True
+
+    def _load_local_model(self) -> None:
+        """Load sentence-transformers model for local embedding."""
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError:
+            raise ImportError(
+                "sentence-transformers required: "
+                "pip install sentence-transformers"
+            )
+        model = SentenceTransformer(
+            self._config.model_name, trust_remote_code=False,
+        )
+        actual_dim = model.get_sentence_embedding_dimension()
+        if actual_dim != self._config.dimension:
+            raise DimensionMismatchError(
+                f"Model '{self._config.model_name}' produces {actual_dim}-dim "
+                f"embeddings but config expects {self._config.dimension}-dim"
+            )
+        self._model = model
+        logger.info(
+            "EmbeddingService: local model loaded (%s, %d-dim)",
+            self._config.model_name,
+            actual_dim,
+        )
+
+    # ------------------------------------------------------------------
+    # Internals — encoding
+    # ------------------------------------------------------------------
+
+    def _encode_single(self, text: str) -> NDArray[np.float32]:
+        """Encode one text. Dispatches to local or cloud."""
+        self._ensure_loaded()
+        if self._config.is_cloud:
+            return self._cloud_embed([text])[0]
+        return self._local_embed_batch([text])[0]
+
+    def _encode_batch(self, texts: list[str]) -> list[NDArray[np.float32]]:
+        """Encode a batch. Dispatches to local or cloud."""
+        self._ensure_loaded()
+        if self._config.is_cloud:
+            return self._cloud_embed(texts)
+        return self._local_embed_batch(texts)
+
+    def _local_embed_batch(
+        self,
+        texts: list[str],
+    ) -> list[NDArray[np.float32]]:
+        """Encode via local sentence-transformers (L2-normalized)."""
+        if self._model is None:
+            raise RuntimeError("Local model not loaded")
+        vecs = self._model.encode(texts, normalize_embeddings=True)
+        if isinstance(vecs, np.ndarray) and vecs.ndim == 2:
+            return [vecs[i] for i in range(vecs.shape[0])]
+        return [np.asarray(v, dtype=np.float32) for v in vecs]
+
+    def _cloud_embed(
+        self,
+        texts: list[str],
+        *,
+        max_retries: int = 3,
+    ) -> list[NDArray[np.float32]]:
+        """Encode via Azure OpenAI embedding API with retry logic.
+
+        Raises on failure — NEVER falls back to local model.
+        """
+        import httpx
+
+        url = (
+            f"{self._config.api_endpoint.rstrip('/')}/openai/deployments/"
+            f"{self._config.deployment_name}/embeddings"
+            f"?api-version={self._config.api_version}"
+        )
+        headers = {
+            "Content-Type": "application/json",
+            "api-key": self._config.api_key,
+        }
+        body = {"input": texts, "model": self._config.deployment_name}
+
+        last_error: Exception | None = None
+        for attempt in range(max_retries):
+            try:
+                with httpx.Client(timeout=httpx.Timeout(30.0)) as client:
+                    resp = client.post(url, headers=headers, json=body)
+                    resp.raise_for_status()
+                data = resp.json()
+                results: list[NDArray[np.float32]] = []
+                for item in sorted(data["data"], key=lambda d: d["index"]):
+                    vec = np.asarray(item["embedding"], dtype=np.float32)
+                    results.append(vec)
+                return results
+            except Exception as exc:
+                last_error = exc
+                wait = 2 ** attempt  # 1s, 2s, 4s
+                logger.warning(
+                    "Cloud embed attempt %d/%d failed: %s (retry in %ds)",
+                    attempt + 1,
+                    max_retries,
+                    exc,
+                    wait,
+                )
+                if attempt < max_retries - 1:
+                    time.sleep(wait)
+
+        raise RuntimeError(
+            f"Cloud embedding failed after {max_retries} attempts: "
+            f"{last_error}"
+        )
+
+    # ------------------------------------------------------------------
+    # Validation
+    # ------------------------------------------------------------------
+
+    def _validate_dimension(self, vec: NDArray) -> None:
+        """Hard validation — crash on mismatch, never silently fall back."""
+        actual = len(vec)
+        if actual != self._config.dimension:
+            raise DimensionMismatchError(
+                f"Embedding dimension {actual} != "
+                f"expected {self._config.dimension}. "
+                f"This is a HARD failure — check your model/API config."
+            )