superlocalmemory 3.4.43 → 3.4.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superlocalmemory",
-  "version": "3.4.43",
+  "version": "3.4.44",
   "description": "Information-geometric agent memory with mathematical guarantees. 4-channel retrieval, Fisher-Rao similarity, zero-LLM mode, EU AI Act compliant. Works with Claude, Cursor, Windsurf, and 17+ AI tools.",
   "keywords": [
     "ai-memory",

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "superlocalmemory"
-version = "3.4.43"
+version = "3.4.44"
 description = "Information-geometric agent memory with mathematical guarantees"
 readme = "README.md"
 license = {text = "AGPL-3.0-or-later"}
@@ -29,48 +29,42 @@ classifiers = [
 ]
 
 dependencies = [
-    "httpx>=0.24.0",
-    "numpy>=1.26.0,<3.0.0",
-    "scipy>=1.12.0,<2.0.0",
-    "networkx>=3.0",
-    "mcp>=1.0.0",
-    "python-dateutil>=2.9.0.post0",
-    "rank-bm25>=0.2.2",
-    "vadersentiment>=3.3.2",
-    "einops>=0.8.2",
-    "fastapi[all]>=0.135.1",
-    "uvicorn>=0.42.0",
-    "websockets>=16.0",
-    "lightgbm>=4.0.0",
-    "orjson>=3.9.0",
-    # CodeGraph — code knowledge graph (v3.4)
-    "tree-sitter>=0.23.0,<1",
-    "tree-sitter-language-pack>=0.5,<1",
-    "rustworkx>=0.15,<1",
-    "watchdog>=4.0,<6",
-    # V3.4.3: Unified Brain
-    "psutil>=5.9.0",
-    "structlog>=24.0.0,<27.0.0",
-    # Cross-platform file locking for single-daemon enforcement.
-    "portalocker>=2.7.0,<4.0.0",
-    # V3.4.18: Semantic search + cross-encoder reranker (npm install parity).
-    # Previously under [search] extra — pip users silently lost 30pp of recall
-    # quality vs. npm users. Now ships by default for both install paths.
-    # v3.4.43: bumped from >=5.0.0 to >=5.2.0 so the resolver doesn't pick
-    # 5.0.0-5.1.2 which cap transformers<5.0.0 (security headroom for when
-    # optimum-onnx upstream eventually supports transformers 5.x).
-    "sentence-transformers[onnx]>=5.2.0",
-    # NOTE on CVE-2026-1839 (transformers Trainer.torch.load RCE):
-    # SLM does NOT use transformers.Trainer (inference-only path via
-    # sentence-transformers + ONNX backend). The vulnerable method
-    # Trainer._load_rng_state is never called by SLM code, so the CVE is
-    # unreachable through SLM's API surface. We CANNOT pin transformers>=5.0.0
-    # because optimum-onnx 0.1.0 (latest upstream) caps transformers<4.58.0
-    # and SLM's embedding_worker.py:68 hard-codes backend="onnx". Will
-    # tighten this pin in a future release once optimum-onnx ships a
-    # transformers-5.x-compatible build.
-    "torch>=2.2.0",
-    "scikit-learn>=1.3.0,<2.0.0",
+    # All versions hard-pinned to the verified-good combination.
+    # Mixing versions outside these pins triggers per-batch memory
+    # blow-up in the embedding worker on Apple Silicon and breaks
+    # recall/remember latency targets. Update only after benchmarking.
+    "httpx==0.28.1",
+    "numpy==2.4.4",
+    "scipy==1.17.1",
+    "networkx==3.6.1",
+    "mcp==1.27.1",
+    "python-dateutil==2.9.0.post0",
+    "rank-bm25==0.2.2",
+    "vadersentiment==3.3.2",
+    "einops==0.8.2",
+    "fastapi[all]==0.136.1",
+    "uvicorn==0.46.0",
+    "websockets==16.0",
+    "lightgbm==4.6.0",
+    "orjson==3.11.9",
+    "tree-sitter==0.25.2",
+    "tree-sitter-language-pack==0.13.0",
+    "rustworkx==0.17.1",
+    "watchdog==5.0.3",
+    "psutil==7.2.2",
+    "structlog==25.5.0",
+    "portalocker==3.2.0",
+    # Semantic search + cross-encoder reranker. Embedding stack is
+    # extremely sensitive to version drift on Apple Silicon — newer
+    # versions allocate dramatically more per-batch memory.
+    "sentence-transformers[onnx]==5.3.0",
+    "onnxruntime==1.24.4",
+    "transformers==4.57.6",
+    "huggingface_hub==0.36.2",
+    "torch==2.11.0",
+    "scikit-learn==1.8.0",
+    # Vector KNN extension for the semantic channel.
+    "sqlite-vec==0.1.9",
 ]
 
 [project.optional-dependencies]
@@ -78,12 +72,13 @@ dependencies = [
 # moved into core in v3.4.18. ``pip install superlocalmemory[search]`` still
 # works but installs nothing extra.
 search = [
-    "sentence-transformers[onnx]>=5.2.0",
-    "einops>=0.8.2",
-    "torch>=2.2.0",
-    "scikit-learn>=1.3.0,<2.0.0",
+    # Same hard pin as core deps — see comment above.
+    "sentence-transformers[onnx]==5.3.0",
+    "einops==0.8.2",
+    "torch==2.11.0",
+    "scikit-learn==1.8.0",
     "geoopt>=0.5.0",
-    "onnxruntime>=1.17.0",
+    "onnxruntime==1.24.4",
 ]
 ui = [
     "fastapi[all]>=0.135.1",

package/scripts/install.ps1

@@ -233,22 +233,31 @@ print('Database ready')
     Write-Host "WARNING: setup_validator.py not found, skipping database init" -ForegroundColor Yellow
 }
 
-# Install core dependencies (required for graph & dashboard)
+# Install SuperLocalMemory and all dependencies via pyproject.toml (single source of truth)
 Write-Host ""
-Write-Host "Installing core dependencies..."
-Write-Host "INFO: This ensures graph visualization and patterns work out-of-box" -ForegroundColor Yellow
+Write-Host "Installing SuperLocalMemory and all dependencies..."
+Write-Host "INFO: Versions are pinned in pyproject.toml -- same versions for every install path" -ForegroundColor Yellow
+
+# Find pyproject.toml (parent of scripts/ or scripts/ itself)
+$ParentDir = Split-Path -Parent $REPO_DIR
+if (Test-Path (Join-Path $ParentDir "pyproject.toml")) {
+    $ProjRoot = $ParentDir
+} elseif (Test-Path (Join-Path $REPO_DIR "pyproject.toml")) {
+    $ProjRoot = $REPO_DIR
+} else {
+    $ProjRoot = $null
+}
 
-$coreRequirements = Join-Path $REPO_DIR "requirements-core.txt"
-if (Test-Path $coreRequirements) {
+if ($ProjRoot) {
     try {
-        & python -m pip install -q -r $coreRequirements 2>$null
-        Write-Host "OK Core dependencies installed (graph, dashboard, patterns)" -ForegroundColor Green
+        & python -m pip install -q -e $ProjRoot 2>$null
+        Write-Host "OK SuperLocalMemory and all dependencies installed (pinned versions)" -ForegroundColor Green
     } catch {
-        Write-Host "WARNING: Core dependency installation failed. Some features may not work." -ForegroundColor Yellow
-        Write-Host "   Install manually: python -m pip install -r $coreRequirements" -ForegroundColor Yellow
+        Write-Host "WARNING: Dependency installation failed." -ForegroundColor Yellow
+        Write-Host "   Install manually: python -m pip install -e $ProjRoot" -ForegroundColor Yellow
     }
 } else {
-    Write-Host "WARNING: requirements-core.txt not found, skipping dependency installation" -ForegroundColor Yellow
+    Write-Host "WARNING: pyproject.toml not found, cannot install dependencies" -ForegroundColor Yellow
 }
 
 # Initialize knowledge graph and pattern learning

package/scripts/install.sh

@@ -358,8 +358,8 @@ except Exception as e:
 
 # Install core dependencies (required for graph & dashboard)
 echo ""
-echo "Installing core dependencies..."
-echo "⏳ This ensures graph visualization and patterns work out-of-box"
+echo "Installing SuperLocalMemory and all dependencies..."
+echo "⏳ Versions are pinned in pyproject.toml — same versions for every install path"
 
 # Detect pip installation method
 if pip3 install --help | grep -q "break-system-packages"; then
@@ -368,31 +368,25 @@ else
     PIP_FLAGS=""
 fi
 
-if [ -f "${REPO_DIR}/requirements-core.txt" ]; then
-    if pip3 install $PIP_FLAGS -q -r "${REPO_DIR}/requirements-core.txt"; then
-        echo "✓ Core dependencies installed (graph, dashboard, patterns)"
-    else
-        echo "⚠️  Core dependency installation failed. Some features may not work."
-        echo "   Install manually: pip3 install -r ${REPO_DIR}/requirements-core.txt"
-    fi
+# Find the repo root (parent of scripts/)
+PKG_ROOT="$(cd "${REPO_DIR}/.." && pwd)"
+if [ -f "${PKG_ROOT}/pyproject.toml" ]; then
+    PROJ_ROOT="${PKG_ROOT}"
+elif [ -f "${REPO_DIR}/pyproject.toml" ]; then
+    PROJ_ROOT="${REPO_DIR}"
 else
-    echo "⚠️  requirements-core.txt not found, skipping dependency installation"
+    PROJ_ROOT=""
 fi
 
-# Install learning dependencies (v2.7+)
-echo ""
-echo "Installing learning dependencies..."
-echo "  Enables intelligent pattern learning and personalized recall"
-
-if [ -f "${REPO_DIR}/requirements-learning.txt" ]; then
-    if pip3 install $PIP_FLAGS -q -r "${REPO_DIR}/requirements-learning.txt" 2>/dev/null; then
-        echo "✓ Learning dependencies installed (personalized ranking enabled)"
+if [ -n "${PROJ_ROOT}" ]; then
+    if pip3 install $PIP_FLAGS -q -e "${PROJ_ROOT}"; then
+        echo "✓ SuperLocalMemory and all dependencies installed (pinned versions)"
     else
-        echo "○ Learning dependencies skipped (core features unaffected)"
-        echo "  To install later: pip3 install lightgbm scipy"
+        echo "⚠️  Dependency installation failed."
+        echo "   Install manually: pip3 install -e ${PROJ_ROOT}"
     fi
 else
-    echo "○ requirements-learning.txt not found (learning features will use rule-based ranking)"
+    echo "⚠️  pyproject.toml not found, cannot install dependencies"
 fi
 
 # Initialize knowledge graph and pattern learning

package/scripts/postinstall.js

@@ -97,83 +97,15 @@ function pipInstall(packages, label) {
     return false;
 }
 
-// Core dependencies (REQUIRED — product won't work without these)
-const coreDeps = [
-    'numpy>=1.26.0', 'scipy>=1.12.0', 'networkx>=3.0',
-    'httpx>=0.24.0', 'python-dateutil>=2.9.0',
-    'rank-bm25>=0.2.2', 'vaderSentiment>=3.3.2',
-    'einops>=0.8.2', 'mcp>=1.0.0',
-];
-
-if (pipInstall(coreDeps, 'core')) {
-    console.log('✓ Core dependencies installed (math, search, NLP)');
-} else {
-    console.log('⚠ Core dependency installation failed.');
-    console.log('  Run manually: pip install ' + coreDeps.join(' '));
-}
-
-// Search + ONNX reranking (V3.3.2 — enables 6-channel retrieval + cross-encoder)
-const searchDeps = [
-    'sentence-transformers[onnx]>=4.0.0',
-    'einops>=0.7.0', 'geoopt>=0.5.0',
-    'onnxruntime>=1.17.0',
-];
-
-console.log('\nInstalling semantic search + ONNX reranking engine...');
-console.log('  (sentence-transformers 4+, ONNX Runtime, Fisher-Rao geometry)');
-if (pipInstall(searchDeps, 'search')) {
-    console.log('✓ Search engine installed (sentence-transformers + ONNX + Fisher-Rao)');
-    console.log('  Cross-encoder reranking enabled for ALL modes (+30pp quality)');
-    console.log('');
-    console.log('  Models auto-download on first use:');
-    console.log('    - Embedding: nomic-ai/nomic-embed-text-v1.5 (~500MB)');
-    console.log('    - Reranker: cross-encoder/ms-marco-MiniLM-L-6-v2 (~90MB)');
-    console.log('  To pre-download now, run: slm warmup');
-} else {
-    console.log('⚠ Search engine installation failed (BM25 keyword search still works).');
-    console.log('  For full 6-channel retrieval + reranking, run:');
-    console.log('  pip install "sentence-transformers[onnx]>=4.0.0" einops geoopt onnxruntime');
-}
-
-// Dashboard dependencies (IMPORTANT — enables web dashboard + MCP server)
-const dashboardDeps = ['fastapi[all]>=0.135.1', 'uvicorn>=0.42.0', 'websockets>=16.0'];
-console.log('\nInstalling dashboard & server dependencies...');
-if (pipInstall(dashboardDeps, 'dashboard')) {
-    console.log('✓ Dashboard & MCP server dependencies installed (fastapi + uvicorn)');
-} else {
-    console.log('⚠ Dashboard installation failed.');
-    console.log('  Run manually: pip install \'fastapi[all]\' uvicorn websockets');
-}
-
-// Learning dependencies (enables adaptive retrieval after 200+ signals)
-const learningDeps = ['lightgbm>=4.0.0'];
-console.log('\nInstalling learning engine...');
-if (pipInstall(learningDeps, 'learning')) {
-    console.log('✓ Learning engine installed (lightgbm — adaptive ranking)');
-} else {
-    console.log('⚠ Learning installation failed (retrieval still works without it).');
-    console.log('  Run manually: pip install lightgbm');
-}
-
-// Performance dependencies (optional — improves caching and JSON speed)
-const perfDeps = ['diskcache>=5.6.0', 'orjson>=3.9.0'];
-console.log('\nInstalling performance optimizations...');
-if (pipInstall(perfDeps, 'performance')) {
-    console.log('✓ Performance optimizations installed (diskcache + orjson)');
-} else {
-    console.log('⚠ Performance deps skipped (system works fine without them).');
-}
-
-// V3.4.3: Unified Brain dependencies (health monitor, structured logging, file watching)
-const brainDeps = ['psutil>=5.9.0', 'structlog>=24.0.0', 'watchdog>=4.0.0'];
-console.log('\nInstalling Unified Brain dependencies (health monitor, file watcher)...');
-if (pipInstall(brainDeps, 'brain')) {
-    console.log('✓ Unified Brain deps installed (psutil + structlog + watchdog)');
-    console.log('  Health monitoring, structured logging, and file watching enabled');
-} else {
-    console.log('⚠ Unified Brain deps partially installed (health monitoring may be limited).');
-    console.log('  Run manually: pip install psutil structlog watchdog');
-}
+// Install the superlocalmemory package and all its pinned dependencies
+// in one shot. pyproject.toml is the single source of truth for versions,
+// so users via npm get exactly the same dep set as users via pip.
+console.log('\nInstalling SuperLocalMemory and all dependencies...');
+console.log('  (Single pip install — versions pinned in pyproject.toml)');
+console.log('  This may take 1-3 minutes (downloads ~500MB of models on first use).');
+console.log('');
+console.log('  Includes: numpy, scipy, fastapi, sentence-transformers, onnxruntime,');
+console.log('           torch, transformers, sqlite-vec, lightgbm, mcp, and more.');
 
 // --- Step 3b: Install the superlocalmemory package itself ---
 // This ensures `python -m superlocalmemory.cli.main` always resolves the

package/src/superlocalmemory/__init__.py

@@ -1,3 +1,3 @@
 """SuperLocalMemory — information-geometric agent memory."""
 
-__version__ = "3.4.39"
+__version__ = "3.4.44"

package/src/superlocalmemory/core/embedding_worker.py

@@ -53,24 +53,25 @@ def _start_parent_watchdog() -> None:
 
 
 def _load_embedding_model(name: str) -> tuple:
-    """Load embedding model. ONNX first (no memory leak), PyTorch fallback.
-
-    V3.3.17: PyTorch SentenceTransformer on ARM64 Mac leaks memory —
-    grows from 300MB to 17GB after ~200 encode calls. ONNX Runtime
-    has no such issue. Same approach as CrossEncoder ONNX migration.
+    """Load embedding model. ONNX CPU-only first, PyTorch fallback.
 
     Returns (model, backend_name) or (None, "").
     """
     from sentence_transformers import SentenceTransformer
 
-    # Tier 1: ONNX (stable memory; ~1.1 GB for nomic-embed-text-v1.5)
+    # ONNX with explicit CPU provider — avoids CoreML EP memory overhead.
     try:
-        m = SentenceTransformer(name, backend="onnx", trust_remote_code=True)
+        m = SentenceTransformer(
+            name,
+            backend="onnx",
+            trust_remote_code=True,
+            model_kwargs={"provider": "CPUExecutionProvider"},
+        )
         return m, "onnx"
     except Exception:
         pass
 
-    # Tier 2: PyTorch CPU (stable at ~1.4GB after 100+ calls, verified)
+    # PyTorch CPU fallback.
     try:
         import torch
         with torch.inference_mode():

package/src/superlocalmemory/core/engine_wiring.py

@@ -78,38 +78,19 @@ def init_embedder(config: SLMConfig) -> Any | None:
     emb_cfg = config.embedding
     provider = emb_cfg.provider
 
-    # --- Explicit ollama provider ---
-    # V3.3.27: HYBRID MODE B — use sentence-transformers subprocess for
-    # embeddings (fast, batched, ~2s) instead of Ollama HTTP per-call (~30s).
-    # Ollama is still used for LLM operations (fact extraction, context
-    # generation) via llm/backbone.py — that path is unchanged.
-    #
-    # Why: The store pipeline calls embed() 200+ times per remember
-    # (scene_builder, type_router, consolidator, entropy_gate, etc.).
-    # Ollama HTTP: 200 * 45ms = 9s minimum + cold starts.
-    # sentence-transformers subprocess: 200 embeds batched = ~1s.
-    #
-    # The embedding model is the SAME (nomic-embed-text-v1.5, 768d) —
-    # identical vectors, zero quality difference. Only the transport changes.
+    # All modes use sentence-transformers subprocess as primary so the
+    # embedding space matches stored vectors. Ollama is fallback only —
+    # Ollama's nomic-embed-text and sentence-transformers nomic-embed-text-v1.5
+    # produce different vectors, so mixing them against an ST-indexed
+    # corpus degrades semantic recall quality.
     if provider == "ollama":
-        if config.mode == Mode.B:
-            # Mode B hybrid: prefer subprocess embedder (fast, batched)
-            st_emb = _try_service_embedder(EmbeddingService, emb_cfg)
-            if st_emb is not None:
-                logger.info(
-                    "Mode B hybrid: using sentence-transformers subprocess "
-                    "for embeddings (fast batched). Ollama used for LLM only."
-                )
-                return st_emb
-            # Fallback: if subprocess unavailable, use Ollama embeddings
-            logger.info("Mode B: sentence-transformers unavailable, using Ollama embeddings")
-            result = _try_ollama_embedder(emb_cfg)
-            if result is not None:
-                return result
-            return None
-        # Mode A/C with explicit ollama: use Ollama embeddings
+        st_emb = _try_service_embedder(EmbeddingService, emb_cfg)
+        if st_emb is not None:
+            logger.info("Using sentence-transformers subprocess (matches stored embedding space)")
+            return st_emb
         result = _try_ollama_embedder(emb_cfg)
         if result is not None:
+            logger.warning("sentence-transformers unavailable; falling back to Ollama (semantic quality may degrade)")
             return result
         return None
 

package/src/superlocalmemory/server/unified_daemon.py

@@ -1342,7 +1342,7 @@ def _start_memory_watchdog() -> None:
     """
     import threading
 
-    MAX_WORKER_MB = 1800  # V3.4.37: 1.8GB — ONNX nomic-embed is ~1.7GB loaded
+    MAX_WORKER_MB = int(os.environ.get("SLM_MAX_WORKER_MB", "2500"))
 
     def watchdog_loop():
         while True:
@@ -1418,12 +1418,23 @@ def _start_pending_materializer() -> None:
                 for item in pending:
                     if _materializer_stop.is_set():
                         break
-                    # Yield to recalls: wait until none in flight
                     waits = 0
                     while _recalls_in_flight() > 0 and waits < 60:
                         time.sleep(0.5)
                         waits += 1
                     try:
+                        import hashlib
+                        content = item["content"]
+                        # Dedup: skip if identical content already stored.
+                        content_hash = hashlib.md5(content.encode()).hexdigest()
+                        dup = engine._db.execute(
+                            "SELECT 1 FROM atomic_facts WHERE "
+                            "content = ? LIMIT 1",
+                            (content,),
+                        )
+                        if dup:
+                            mark_done(item["id"])
+                            continue
                         import json as _json
                         md_str = item.get("metadata") or "{}"
                         try:
@@ -1432,7 +1443,29 @@ def _start_pending_materializer() -> None:
                             md = {}
                         if item.get("tags"):
                             md.setdefault("tags", item["tags"])
-                        engine.store(item["content"], metadata=md)
+                        # Create memory row (FK target for atomic_facts)
+                        from datetime import datetime, timezone
+                        from superlocalmemory.storage.models import (
+                            AtomicFact, FactType,
+                        )
+                        mem_id = content_hash[:16]
+                        engine._db.execute(
+                            "INSERT OR IGNORE INTO memories "
+                            "(memory_id, profile_id, content, "
+                            "session_id, speaker, role, created_at, "
+                            "metadata_json) VALUES (?,?,?,?,?,?,?,?)",
+                            (mem_id, engine._profile_id, content,
+                             "", "", "user",
+                             datetime.now(timezone.utc).isoformat(),
+                             _json.dumps(md)),
+                        )
+                        fact = AtomicFact(
+                            content=content,
+                            fact_type=FactType.EPISODIC,
+                            memory_id=mem_id,
+                            profile_id=engine._profile_id,
+                        )
+                        engine.store_fact_direct(fact)
                         mark_done(item["id"])
                     except Exception as exc:
                         logger.warning(