superlocalmemory 3.6.23 → 3.7.0

package/src/superlocalmemory/core/backend_orchestrator.py

@@ -79,11 +79,11 @@ class BackendOrchestrator:
         # 1. Apply schema (if not already applied)
         self._apply_schema_v345()
 
-        # 2. Initialize TierManager (always)
+        # 2. Initialize TierManager (always). Backends are refreshed after
+        # optional projections have been opened below.
         try:
-            from superlocalmemory.core.tier_manager import evaluate_tiers, set_backends
+            from superlocalmemory.core.tier_manager import evaluate_tiers
             self._tiers = evaluate_tiers
-            set_backends(cozo=self._cozo, lancedb=self._lancedb)
             logger.info("BackendOrchestrator: TierManager initialized")
         except Exception as exc:
             logger.warning("TierManager init failed (non-fatal): %s", exc)
@@ -97,6 +97,16 @@ class BackendOrchestrator:
         except Exception as exc:
             logger.warning("Initial rebalance failed (non-fatal): %s", exc)
 
+        # Backends may be installed with the product, but installing a wheel
+        # is not authorization to mutate an existing data root.  Only a
+        # verified, explicit promotion may initialize and migrate projections.
+        if getattr(self._config, "scale_engine_state", "local_core") != "promoted":
+            logger.info(
+                "Scale Engine remains on Local Core (state=%s)",
+                getattr(self._config, "scale_engine_state", "local_core"),
+            )
+            return
+
         # 4. Initialize CozoDB if available
         cozo_available = self._detect_cozo()
         if cozo_available:
@@ -107,20 +117,18 @@ class BackendOrchestrator:
         if lancedb_available:
             self._init_lancedb()
 
-        # 6. Auto-migrate
+        # A promoted stage is already parity-verified.  Never rebuild it at
+        # startup: automatic migration would bypass the staged lifecycle and
+        # could make the active projection diverge from canonical SQLite.
         if self._cozo:
-            status = self._cozo_status()
-            if status in ("not_initialized", "migrating"):
-                if status == "migrating":
-                    logger.warning("CozoDB migration interrupted — rebuilding")
-                self._migrate_cozo()
-
+            self._update_status("cozo", "active", self._cozo.health_check().get("edges", 0))
         if self._lancedb:
-            status = self._lancedb_status()
-            if status in ("not_initialized", "migrating"):
-                if status == "migrating":
-                    logger.warning("LanceDB migration interrupted — rebuilding")
-                self._migrate_lancedb()
+            self._update_status("lancedb", "active", self._lancedb.health_check().get("vectors", 0))
+        try:
+            from superlocalmemory.core.tier_manager import set_backends
+            set_backends(cozo=self._cozo, lancedb=self._lancedb)
+        except Exception as exc:
+            logger.warning("TierManager backend registration failed (non-fatal): %s", exc)
 
         logger.info("BackendOrchestrator: daemon ready (cozo=%s, lancedb=%s)",
                      "active" if self._cozo and self._cozo_status() == "active" else "off",
@@ -149,15 +157,38 @@ class BackendOrchestrator:
                 self._sync_fact_embedding(fact)
 
     def _sync_fact_entities(self, fact: Any) -> None:
-        """Sync fact's entities and edges to CozoDB."""
+        """Synchronize one fact's canonical entity bridge and fact edges."""
         try:
+            # Retrying ingestion must not retain stale fact/entity links.
+            self._cozo.remove_fact(fact.fact_id)
             entities = getattr(fact, "canonical_entities", []) or []
+            profile_id = getattr(fact, "profile_id", "default") or "default"
             for eid in entities:
-                self._cozo.add_entity(eid, eid, "concept", {})
-                # Add edges from this entity to existing ones
-                for other in entities:
-                    if other != eid:
-                        self._cozo.add_edge(eid, other, "co_occurs", 1.0)
+                rows = self._db.execute(
+                    "SELECT canonical_name, entity_type, fact_count FROM canonical_entities "
+                    "WHERE entity_id = ? AND profile_id = ?",
+                    (eid, profile_id),
+                )
+                if rows:
+                    entity = dict(rows[0])
+                    self._cozo.add_entity(
+                        eid,
+                        entity.get("canonical_name") or eid,
+                        entity.get("entity_type") or "concept",
+                        {"fact_count": int(entity.get("fact_count") or 0)},
+                        profile_id,
+                    )
+            self._cozo.add_fact_entities(fact.fact_id, entities, profile_id)
+            for row in self._db.execute(
+                "SELECT source_id, target_id, edge_type, weight FROM graph_edges "
+                "WHERE profile_id = ? AND (source_id = ? OR target_id = ?)",
+                (profile_id, fact.fact_id, fact.fact_id),
+            ):
+                edge = dict(row)
+                self._cozo.add_edge(
+                    edge["source_id"], edge["target_id"], edge.get("edge_type") or "related",
+                    float(edge.get("weight") or 1.0), profile_id=profile_id,
+                )
         except Exception as exc:
             logger.debug("CozoDB incremental sync skipped: %s", exc)
 
@@ -169,10 +200,30 @@ class BackendOrchestrator:
                 tier = getattr(fact, "lifecycle", "active")
                 self._lancedb.add_vectors(
                     [fact.fact_id], [embedding], [tier],
+                    getattr(fact, "profile_id", "default") or "default",
                 )
         except Exception as exc:
             logger.debug("LanceDB incremental sync skipped: %s", exc)
 
+    def sync_deleted_fact(self, fact_id: str) -> None:
+        """Remove a fact from derived projections after canonical deletion."""
+        if self._cozo and self._cozo_status() == "active":
+            try:
+                self._cozo.remove_fact(fact_id)
+            except Exception as exc:
+                logger.warning("Cozo deletion sync failed for %s: %s", fact_id[:16], exc)
+        if self._lancedb and self._lancedb_status() == "active":
+            try:
+                self._lancedb.remove_vector(fact_id)
+            except Exception as exc:
+                logger.warning("Lance deletion sync failed for %s: %s", fact_id[:16], exc)
+
+    def sync_changed_fact(self, fact_id: str) -> None:
+        """Refresh projections after an authorized canonical fact update."""
+        fact = self._db.get_fact(fact_id)
+        if fact is not None:
+            self.sync_new_fact(fact)
+
     # ------------------------------------------------------------------
     # Backend Access
     # ------------------------------------------------------------------
@@ -189,6 +240,15 @@ class BackendOrchestrator:
             return self._lancedb
         return None
 
+    def graph_retrieval_ready(self) -> bool:
+        """Whether Cozo can be injected into entity recall.
+
+        Cozo carries both canonical entity mappings and fact graph edges.  The
+        entity channel still shadows every projected result against SQLite and
+        fails closed on any mismatch, so availability never weakens recall.
+        """
+        return bool(self._cozo and self._cozo_status() == "active")
+
     # ------------------------------------------------------------------
     # Health Check
     # ------------------------------------------------------------------

package/src/superlocalmemory/core/config.py

@@ -13,13 +13,14 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 
-logger = logging.getLogger(__name__)
-
+from superlocalmemory.infra.data_root import DynamicStatePath, canonical_data_root
 from superlocalmemory.storage.models import Mode
 
+logger = logging.getLogger(__name__)
+
 
 # ---------------------------------------------------------------------------
 # Canonical limits (WP-02 — single source of truth across all surfaces)
@@ -35,7 +36,7 @@ CANONICAL_RECALL_LIMIT: int = 20
 # Default Paths
 # ---------------------------------------------------------------------------
 
-DEFAULT_BASE_DIR = Path.home() / ".superlocalmemory"
+DEFAULT_BASE_DIR = DynamicStatePath()
 DEFAULT_DB_NAME = "memory.db"
 DEFAULT_PROFILES_FILE = "profiles.json"
 CURRENT_MODE_FILE = "current_mode"
@@ -43,6 +44,11 @@ CURRENT_MODE_FILE = "current_mode"
 _MODE_CONFIG_NAMES: dict | None = None
 
 
+def _runtime_base_dir() -> Path:
+    """Resolve the process namespace when a config root is not explicit."""
+    return canonical_data_root()
+
+
 # ---------------------------------------------------------------------------
 # Embedding Config
 # ---------------------------------------------------------------------------
@@ -768,7 +774,7 @@ class SLMConfig:
     """
 
     mode: Mode = Mode.A
-    base_dir: Path = DEFAULT_BASE_DIR
+    base_dir: Path = field(default_factory=_runtime_base_dir)
     db_path: Path | None = None    # Computed from base_dir if None
     active_profile: str = "default"
 
@@ -801,6 +807,9 @@ class SLMConfig:
     # v3.5.0: scaling backends — "sqlite" / "cozo" / "auto" / "lancedb" / "sqlite-vec" / "auto".
     graph_backend: str = "auto"       # "auto" = cozo if pycozo installed, else sqlite
     vector_backend: str = "auto"      # "auto" = lancedb if installed, else sqlite-vec
+    # Scale Engine is installed separately from activation.  Existing roots
+    # stay on SQLite until a staged parity check promotes these projections.
+    scale_engine_state: str = "local_core"  # local_core | prepared | verified | promoted
     evolution: EvolutionConfig = field(default_factory=EvolutionConfig)
     health: HealthConfig = field(default_factory=HealthConfig)
 
@@ -824,16 +833,22 @@ class SLMConfig:
     @classmethod
     def load(cls, config_path: Path | None = None) -> SLMConfig:
         """Load config from JSON file. Returns default Mode A if file doesn't exist."""
-        # WP-07: resolve base dir through slm_home() at call time so all 3 env
-        # aliases (SLM_DATA_DIR → SL_MEMORY_PATH → SLM_HOME) are honoured.
+        # V3.7: process namespace selection is authoritative. A persisted
+        # base_dir may bootstrap a legacy custom root only when no environment
+        # alias selected the process namespace.
         try:
-            from superlocalmemory.cli._lazy_init import slm_home as _slm_home
-            _runtime_base = _slm_home()
+            from superlocalmemory.infra.data_root import (
+                canonical_data_root,
+                environment_data_root,
+            )
+            _environment_base = environment_data_root()
+            _runtime_base = canonical_data_root()
         except Exception:
+            _environment_base = None
             _runtime_base = DEFAULT_BASE_DIR
         path = config_path or (_runtime_base / "config.json")
         if not path.exists():
-            return cls.for_mode(Mode.A)
+            return cls.for_mode(Mode.A, base_dir=_runtime_base)
         import json
         try:
             data = json.loads(path.read_text())
@@ -843,13 +858,21 @@ class SLMConfig:
             logger.warning(
                 "config.json unreadable/corrupt (%s) — using Mode A default", exc
             )
-            return cls.for_mode(Mode.A)
+            return cls.for_mode(Mode.A, base_dir=_runtime_base)
         mode = Mode(data.get("mode", "a"))
         llm_data = data.get("llm", {})
         emb_data = data.get("embedding", {})
         # V3.5.9: read base_dir before constructing config so for_mode() builds
         # db_path from the user's directory, not DEFAULT_BASE_DIR.
-        raw_base_dir = Path(data.get("base_dir", str(_runtime_base)))
+        if _environment_base is not None:
+            raw_base_dir = _environment_base
+        else:
+            try:
+                raw_base_dir = canonical_data_root(
+                    configured_base_dir=data.get("base_dir", _runtime_base),
+                )
+            except Exception:
+                raw_base_dir = _runtime_base
         config = cls.for_mode(
             mode,
             llm_provider=llm_data.get("provider", ""),
@@ -865,6 +888,13 @@ class SLMConfig:
             base_dir=raw_base_dir,
         )
         config.active_profile = data.get("active_profile", "default")
+        config.graph_backend = data.get("graph_backend", "auto")
+        config.vector_backend = data.get("vector_backend", "auto")
+        state = data.get("scale_engine_state", "local_core")
+        config.scale_engine_state = (
+            state if state in {"local_core", "prepared", "verified", "promoted"}
+            else "local_core"
+        )
 
         # V3.3 config fields (additive — defaults work if missing from JSON)
         fg = data.get("forgetting", {})
@@ -1011,6 +1041,9 @@ class SLMConfig:
         data = {
             "mode": effective_mode,
             "active_profile": self.active_profile,
+            "graph_backend": self.graph_backend,
+            "vector_backend": self.vector_backend,
+            "scale_engine_state": self.scale_engine_state,
             "base_dir": str(self.base_dir),  # V3.5.9: persist so load() can restore custom paths
             "llm": {
                 "provider": self.llm.provider,
@@ -1026,11 +1059,10 @@ class SLMConfig:
                 "api_key": self.embedding.api_key,
                 "deployment_name": self.embedding.deployment_name,
             },
-            "retrieval": {
-                "use_cross_encoder": self.retrieval.use_cross_encoder,
-                "cross_encoder_model": self.retrieval.cross_encoder_model,
-                "cross_encoder_backend": self.retrieval.cross_encoder_backend,
-            },
+            # Persist the complete retrieval contract.  Saving only the
+            # cross-encoder subset silently reset channel limits, evidence
+            # floors, and agentic settings on the next mode/provider change.
+            "retrieval": asdict(self.retrieval),
         }
 
         # V3.4.11: Persist evolution config (C-CONFIGSAVE fix)
@@ -1165,6 +1197,7 @@ class SLMConfig:
                 base_dir=_base,
                 embedding=_a_emb,
                 llm=LLMConfig(),  # No LLM
+                temporal_validator=TemporalValidatorConfig(mode="a"),
                 retrieval=RetrievalConfig(
                     # V3.3.2: ONNX cross-encoder enabled for all modes (~200MB)
                     use_cross_encoder=True,
@@ -1206,6 +1239,7 @@ class SLMConfig:
                     api_base=llm_api_base or "http://localhost:11434",
                     api_key=llm_api_key or "",
                 ),
+                temporal_validator=TemporalValidatorConfig(mode="b"),
                 retrieval=RetrievalConfig(
                     # V3.3.2: ONNX cross-encoder enabled for all modes (~200MB)
                     use_cross_encoder=True,
@@ -1226,14 +1260,24 @@ class SLMConfig:
                 api_endpoint=embedding_endpoint,
                 api_key=embedding_key,
             )
-        else:
+        elif embedding_endpoint:
             _c_emb = EmbeddingConfig(
-                model_name="text-embedding-3-large",
-                dimension=3072,
+                model_name=embedding_model_name or "text-embedding-3-large",
+                dimension=embedding_dimension or 3072,
                 api_endpoint=embedding_endpoint,
                 api_key=embedding_key,
                 deployment_name=embedding_deployment,
             )
+        else:
+            # Mode C upgrades LLM capability; it must not silently require a
+            # second paid embedding provider.  A 3072-dim cloud embedding
+            # contract without an endpoint/key was auto-routed to the local
+            # 768-dim Ollama embedder, which made ingestion fail at vector
+            # materialization. Cloud embeddings remain an explicit opt-in.
+            _c_emb = EmbeddingConfig(
+                model_name="nomic-ai/nomic-embed-text-v1.5",
+                dimension=768,
+            )
         return cls(
             mode=mode,
             base_dir=_base,
@@ -1244,6 +1288,7 @@ class SLMConfig:
                 api_key=llm_api_key,
                 api_base=llm_api_base,
             ),
+            temporal_validator=TemporalValidatorConfig(mode="c"),
             channel_weights=ChannelWeights(
                 semantic=1.5,
                 bm25=1.2,

package/src/superlocalmemory/core/consolidation_engine.py

@@ -160,10 +160,9 @@ class ConsolidationEngine:
                             def mine(self, *a, **kw): return []
                     from superlocalmemory.hooks.auto_parameterize import AutoParameterizeHook
                     from superlocalmemory.core.config import ParameterizationConfig
-                    from pathlib import Path as _Path
-
                     p_config = getattr(self._slm_config, "parameterization", ParameterizationConfig())
-                    learning_db = str(_Path.home() / ".superlocalmemory" / "learning.db")
+                    from superlocalmemory.infra.data_root import state_path
+                    learning_db = str(state_path("learning.db"))
                     beh_store = BehavioralPatternStore(learning_db)
                     cross_proj = CrossProjectAggregator(self._db)
                     wf_miner = WorkflowMiner(self._db)
@@ -483,10 +482,13 @@ class ConsolidationEngine:
                 continue
 
             # Promote: active -> warm (lifecycle transition)
-            self._db.execute(
-                "UPDATE atomic_facts SET lifecycle = 'warm' "
-                "WHERE fact_id = ? AND lifecycle = 'active'",
-                (fact_id,),
+            from superlocalmemory.core.lifecycle_state import set_fact_lifecycle_zone
+            set_fact_lifecycle_zone(
+                self._db,
+                [fact_id],
+                "warm",
+                profile_id=profile_id,
+                from_atomic=("active",),
             )
             promoted += 1
 

package/src/superlocalmemory/core/context_cache.py

@@ -27,6 +27,7 @@ import hashlib
 import hmac
 import json
 import os
+import secrets
 import sqlite3
 import time
 from dataclasses import dataclass, field
@@ -38,10 +39,10 @@ from superlocalmemory.core.security_primitives import (
     redact_secrets,
     safe_resolve,
 )
+from superlocalmemory.infra.data_root import DynamicStatePath, canonical_data_root
 
-
-CACHE_DB_DEFAULT: Path = Path.home() / ".superlocalmemory" / "active_brain_cache.db"
-INSTALL_TOKEN_DEFAULT: Path = Path.home() / ".superlocalmemory" / ".install_token"
+CACHE_DB_DEFAULT = DynamicStatePath("active_brain_cache.db")
+INSTALL_TOKEN_DEFAULT = DynamicStatePath(".install_token")
 
 TTL_SECONDS: int = 120
 CLEANUP_HORIZON_SECONDS: int = 600
@@ -90,6 +91,44 @@ def _read_install_token(home: Path) -> str | None:
     return token or None
 
 
+def _ensure_install_token_at(home: Path) -> str:
+    """Create/read the binding token for an explicit cache namespace.
+
+    The shared security primitive owns the canonical namespace. This bounded
+    variant preserves ``home_dir`` as a real override without mutating global
+    path state or routing an explicit cache through the canonical token.
+    """
+    if home.resolve(strict=False) == canonical_data_root():
+        return ensure_install_token()
+
+    token_path = home / ".install_token"
+    try:
+        token = token_path.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        token = ""
+    if token:
+        return token
+
+    token = secrets.token_hex(32)
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        fd = os.open(str(token_path), flags, 0o600)
+        try:
+            os.write(fd, token.encode("utf-8"))
+        finally:
+            os.close(fd)
+    except FileExistsError:
+        existing = token_path.read_text(encoding="utf-8").strip()
+        if existing:
+            return existing
+        raise RuntimeError(f"install token is empty: {token_path}")
+    if os.name != "nt":
+        os.chmod(token_path, 0o600)
+    return token
+
+
 # ---------------------------------------------------------------------------
 # Writer (daemon-side)
 # ---------------------------------------------------------------------------
@@ -108,10 +147,14 @@ class ContextCache:
         db_path: Path | None = None,
         home_dir: Path | None = None,
     ) -> None:
-        self._home = home_dir or (Path.home() / ".superlocalmemory")
+        self._home = Path(home_dir) if home_dir is not None else canonical_data_root()
         self._home.mkdir(parents=True, exist_ok=True)
 
-        raw = db_path or (self._home / "active_brain_cache.db")
+        raw = db_path or (
+            Path(CACHE_DB_DEFAULT)
+            if home_dir is None
+            else self._home / "active_brain_cache.db"
+        )
         self._db_path = safe_resolve(self._home, Path(raw).name) \
             if Path(raw).parent == self._home else \
             safe_resolve(self._home, raw)
@@ -181,7 +224,7 @@ class ContextCache:
             );
             """
         )
-        token = ensure_install_token()
+        token = _ensure_install_token_at(self._home)
         now = int(time.time())
         self._write_conn.execute(
             "INSERT OR IGNORE INTO slm_meta (key, value, created_at) "
@@ -313,12 +356,17 @@ def read_entry_fast(
       - stdlib-only — no heavy imports, no daemon HTTP call.
     """
     try:
-        home = home_dir or (Path.home() / ".superlocalmemory")
+        home = Path(home_dir) if home_dir is not None else canonical_data_root()
         if not home.exists():
             return None
 
         requested = db_path or Path(
-            os.environ.get("SLM_CACHE_DB") or (home / "active_brain_cache.db"),
+            os.environ.get("SLM_CACHE_DB")
+            or (
+                Path(CACHE_DB_DEFAULT)
+                if home_dir is None
+                else home / "active_brain_cache.db"
+            ),
         )
         try:
             resolved = safe_resolve(home, Path(requested).resolve())