superlocalmemory 3.2.2 → 3.3.0

package/src/superlocalmemory/core/config.py

@@ -92,6 +92,7 @@ class ChannelWeights:
     entity_graph: float = 1.3
     temporal: float = 1.0
     spreading_activation: float = 1.0  # Phase 3: 5th channel (BC-08: default value)
+    hopfield: float = 0.8  # Phase G: 6th channel (Hopfield associative memory)
 
     def as_dict(self) -> dict[str, float]:
         return {
@@ -100,6 +101,7 @@ class ChannelWeights:
             "entity_graph": self.entity_graph,
             "temporal": self.temporal,
             "spreading_activation": self.spreading_activation,
+            "hopfield": self.hopfield,
         }
 
 
@@ -162,6 +164,9 @@ class RetrievalConfig:
     spreading_activation_decay: float = 0.7
     spreading_activation_threshold: float = 0.1
 
+    # Hopfield (Phase G: 6th channel)
+    hopfield_top_k: int = 50
+
     # Trust weighting — apply Bayesian trust scores to retrieval ranking.
     # When enabled, each fact's score is multiplied by a trust weight in [0.5, 1.5].
     # Low-trust facts are demoted; high-trust facts are promoted.
@@ -217,10 +222,10 @@ class MathConfig:
 class ConsolidationConfig:
     """Configuration for sleep-time consolidation (Phase 5).
 
-    Feature-flagged: starts enabled=False (Rule 12).
+    Ships enabled by default. Users can disable via slm config.
     """
 
-    enabled: bool = False                   # Feature flag (Rule 12)
+    enabled: bool = True
     step_count_trigger: int = 50            # Lightweight consolidation every N stores (L7)
     session_trigger: bool = True            # Run on session end
     idle_timeout_seconds: int = 300         # 5 min inactivity
@@ -233,14 +238,232 @@ class ConsolidationConfig:
     decay_days_threshold: int = 30          # Edge decay after N days
 
 
+@dataclass(frozen=True)
+class ForgettingConfig:
+    """Ebbinghaus forgetting configuration."""
+
+    enabled: bool = True
+    # Strength coefficients
+    alpha: float = 2.0              # Access frequency weight (log scale)
+    beta: float = 1.5               # Importance weight (PageRank)
+    gamma: float = 1.0              # Confirmation count weight
+    delta: float = 0.5              # Emotional salience weight
+    # Strength bounds
+    min_strength: float = 0.1       # Floor (prevents instant forgetting)
+    max_strength: float = 100.0     # Ceiling (numerical stability)
+    # Zone thresholds
+    archive_threshold: float = 0.2  # Below this -> ARCHIVE
+    forget_threshold: float = 0.05  # Below this -> FORGOTTEN
+    # Spaced repetition
+    learning_rate: float = 1.0      # eta in spaced repetition update
+    # Coupling
+    forgetting_drift_scale: float = 0.5  # How strongly forgetting affects Langevin drift
+    # Scheduler
+    scheduler_interval_minutes: int = 30  # How often to recompute retentions
+    # Immunity
+    core_memory_immune: bool = True  # Core Memory blocks never forget
+
+
+@dataclass(frozen=True)
+class HopfieldConfig:
+    """Modern Continuous Hopfield Network configuration (Ramsauer et al., 2020).
+
+    Energy: E(xi) = -log(sum_i exp(B * xi' * x_i)) + B/2 * ||xi||^2
+    Update: xi_new = X' @ softmax(B * X @ xi)
+    Beta:   B = 1/sqrt(d) where d = dimension
+    Storage capacity: O(e^{d/2}) -- exponential in dimension.
+    """
+
+    enabled: bool = True
+    dimension: int = 768
+    max_iterations: int = 1
+    convergence_epsilon: float = 1e-6
+    prefilter_threshold: int = 10_000
+    prefilter_candidates: int = 1000
+    skip_threshold: int = 100_000
+    cache_ttl_seconds: float = 60.0
+
+
+@dataclass(frozen=True)
+class ReaperConfig:
+    """Process health & stale reaper configuration (Phase H0).
+
+    Prevents zombie SLM processes from exhausting RAM.
+    """
+
+    enabled: bool = True
+    heartbeat_interval_seconds: int = 60
+    orphan_age_threshold_hours: float = 4.0
+    pid_file_path: str = ""
+    graceful_timeout_seconds: float = 5.0
+
+
+@dataclass(frozen=True)
+class PolarQuantConfig:
+    """PolarQuant embedding quantization configuration.
+
+    Random orthogonal rotation + recursive polar + scalar quantization.
+    Reference: TurboQuant (ICLR 2026), PolarQuant (arXiv 2502.02617).
+    """
+
+    dimension: int = 768
+    rotation_matrix_path: str = ""  # empty = ~/.superlocalmemory/polar_rotation.npy
+    seed: int = 42                  # reproducible rotation matrix
+
+
+@dataclass(frozen=True)
+class QJLConfig:
+    """QJL 1-bit residual correction configuration.
+
+    Random projection + sign-bit quantization for asymmetric IP estimation.
+    Reference: QJL (AAAI 2025, arXiv 2406.03482).
+    """
+
+    projection_dim: int = 128
+    seed: int = 43  # separate from PolarQuant
+
+
+@dataclass(frozen=True)
+class QuantizationConfig:
+    """Memory-aware embedding quantization (EAP + LP2E).
+
+    Couples Ebbinghaus retention to embedding precision.
+    """
+
+    enabled: bool = True
+    polar: PolarQuantConfig = field(default_factory=PolarQuantConfig)
+    qjl: QJLConfig = field(default_factory=QJLConfig)
+    default_bit_width: int = 32
+    eap_enabled: bool = True
+    keep_float32_backup: bool = True
+    auto_compact_interval_hours: int = 6
+    polar_search_penalty: float = 0.95
+
+
+@dataclass(frozen=True)
+class CCQConfig:
+    """Cognitive Consolidation Quantization configuration (Phase E).
+
+    Ships enabled by default. CCQ runs as Step 7 of the consolidation cycle.
+    Biological analogy: sleep-time hippocampal-neocortical transfer.
+    """
+
+    enabled: bool = True
+
+    # Candidate identification
+    retention_threshold: float = 0.5
+    max_candidates_per_run: int = 200
+
+    # Clustering
+    min_entity_overlap: int = 2
+    temporal_window_days: int = 7
+    min_cluster_size: int = 3
+    max_cluster_size: int = 20
+
+    # Gist extraction
+    use_llm_gist: bool = True
+    max_gist_chars: int = 500
+    min_entity_coverage: float = 0.5
+
+    # Embedding compression
+    target_bit_width: int = 2
+    compress_embeddings: bool = True
+
+    # Scheduling
+    store_count_trigger: int = 100
+    run_on_session_end: bool = True
+
+    # Safety
+    core_memory_immune: bool = True
+
+
+@dataclass(frozen=True)
+class SAGQConfig:
+    """Spreading Activation-Guided Quantization configuration.
+
+    Centrality formula:
+        centrality(i) = w_pagerank * pr_norm + w_degree * deg_norm + w_sa_freq * sa_freq_norm
+
+    SAGQ precision:
+        sagq_bw = b_min + (b_max - b_min) * centrality, snapped to valid_bit_widths
+
+    Combined precision (with Phase A EAP):
+        final_bw = max(eap_bw, sagq_bw)
+    """
+
+    enabled: bool = True
+
+    # Centrality weights (MUST sum to 1.0 -- validated in __post_init__)
+    w_pagerank: float = 0.5      # PageRank structural importance
+    w_degree: float = 0.3        # Degree centrality (connection count)
+    w_sa_freq: float = 0.2       # Spreading activation frequency (7-day window)
+
+    # Bit-width range
+    b_min: int = 2               # Minimum bit-width (most aggressive quantization)
+    b_max: int = 32              # Maximum bit-width (full float32 precision)
+
+    # Valid bit-widths (snapping targets) -- must be sorted ascending
+    valid_bit_widths: tuple[int, ...] = (2, 4, 8, 32)
+
+    # SA frequency window (days to look back in activation_cache)
+    sa_frequency_window_days: int = 7
+
+    # Scheduler
+    scheduler_interval_hours: float = 6.0   # How often to run combined scheduler
+
+    def __post_init__(self) -> None:
+        weight_sum = self.w_pagerank + self.w_degree + self.w_sa_freq
+        if abs(weight_sum - 1.0) > 1e-6:
+            raise ValueError(
+                f"SAGQConfig centrality weights must sum to 1.0, got {weight_sum:.6f}"
+            )
+        if not self.valid_bit_widths:
+            raise ValueError("SAGQConfig.valid_bit_widths must not be empty")
+        if self.b_min < 1:
+            raise ValueError(f"SAGQConfig.b_min must be >= 1, got {self.b_min}")
+        if self.b_max < self.b_min:
+            raise ValueError(
+                f"SAGQConfig.b_max ({self.b_max}) must be >= b_min ({self.b_min})"
+            )
+
+
+@dataclass(frozen=True)
+class ParameterizationConfig:
+    """Soft prompt parameterization configuration (Phase F: The Learning Brain).
+
+    Controls pattern extraction, prompt generation, injection, and lifecycle.
+    Ships enabled by default. Pure text soft prompts — no LoRA, no weights.
+    """
+
+    enabled: bool = True
+
+    # Pattern extraction
+    min_confidence: float = 0.7        # Minimum pattern confidence [0.3, 1.0]
+    min_evidence: int = 5              # Minimum evidence count for behavioral/workflow
+    cross_project_boost: float = 1.2   # 20% confidence boost for cross-project patterns
+
+    # Prompt generation
+    max_prompt_tokens: int = 500       # Token budget for soft prompts
+    max_memory_tokens: int = 1500      # Token budget for regular memories
+    categories_enabled: tuple[str, ...] = (
+        "identity", "tech_preference", "communication_style",
+        "workflow_pattern", "project_context", "decision_history",
+        "avoidance",
+    )
+
+    # Lifecycle
+    refresh_interval_hours: float = 24.0   # Min hours between parameterization runs
+    effectiveness_tracking: bool = True     # Track prompt effectiveness via feedback
+
+
 @dataclass(frozen=True)
 class TemporalValidatorConfig:
     """Configuration for temporal intelligence (Phase 4).
 
-    Feature-flagged: starts enabled=False (Rule 12).
+    Ships enabled by default. Users can disable via slm config.
     """
 
-    enabled: bool = False
+    enabled: bool = True
     mode: str = "a"                              # "a" (sheaf), "b"/"c" (LLM)
 
     # Sheaf contradiction threshold
@@ -263,8 +486,7 @@ class TemporalValidatorConfig:
 class AutoInvokeConfig:
     """Configuration for the Auto-Invoke Engine (Phase 2).
 
-    Feature-flagged: starts enabled=False, graduated to default on
-    after MRR validation passes (Rule 12).
+    Ships enabled by default. Users can disable via slm config.
 
     References:
       - SYNAPSE: FOK gating (fok_threshold = 0.12)
@@ -272,7 +494,7 @@ class AutoInvokeConfig:
       - Zep/Hindsight: multi-signal ranking consensus
     """
 
-    enabled: bool = False                      # Feature flag (Rule 12)
+    enabled: bool = True
     profile_id: str = "default"
 
     # Scoring weights (4-signal default) -- must sum to 1.0
@@ -341,6 +563,15 @@ class SLMConfig:
     consolidation: ConsolidationConfig = field(
         default_factory=ConsolidationConfig,
     )
+    forgetting: ForgettingConfig = field(default_factory=ForgettingConfig)
+    hopfield: HopfieldConfig = field(default_factory=HopfieldConfig)
+    reaper: ReaperConfig = field(default_factory=ReaperConfig)
+    quantization: QuantizationConfig = field(default_factory=QuantizationConfig)
+    sagq: SAGQConfig = field(default_factory=SAGQConfig)
+    ccq: CCQConfig = field(default_factory=CCQConfig)
+    parameterization: ParameterizationConfig = field(
+        default_factory=ParameterizationConfig,
+    )
 
     def __post_init__(self) -> None:
         if self.db_path is None:
@@ -369,6 +600,22 @@ class SLMConfig:
             embedding_deployment=emb_data.get("deployment_name", ""),
         )
         config.active_profile = data.get("active_profile", "default")
+
+        # V3.3 config fields (additive — defaults work if missing from JSON)
+        fg = data.get("forgetting", {})
+        if fg:
+            config.forgetting = ForgettingConfig(**{
+                k: v for k, v in fg.items()
+                if k in ForgettingConfig.__dataclass_fields__
+            })
+
+        rt = data.get("retrieval", {})
+        if rt:
+            config.retrieval = RetrievalConfig(**{
+                k: v for k, v in rt.items()
+                if k in RetrievalConfig.__dataclass_fields__
+            })
+
         return config
 
     def save(self, config_path: Path | None = None) -> None:
@@ -376,6 +623,14 @@ class SLMConfig:
         import json
         path = config_path or (self.base_dir / "config.json")
         path.parent.mkdir(parents=True, exist_ok=True)
+        # Read existing config to preserve V3.3 fields not in this save
+        existing = {}
+        if path.exists():
+            try:
+                existing = json.loads(path.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+
         data = {
             "mode": self.mode.value,
             "active_profile": self.active_profile,
@@ -393,7 +648,16 @@ class SLMConfig:
                 "api_key": self.embedding.api_key,
                 "deployment_name": self.embedding.deployment_name,
             },
+            "retrieval": {
+                "use_cross_encoder": self.retrieval.use_cross_encoder,
+            },
         }
+
+        # Preserve existing V3.3 config sections that aren't in for_mode()
+        for key in ("forgetting", "quantization", "sagq", "embedding_signature", "auto_invoke"):
+            if key in existing:
+                data[key] = existing[key]
+
         path.write_text(json.dumps(data, indent=2))
 
     @staticmethod
@@ -456,11 +720,13 @@ class SLMConfig:
                 embedding=EmbeddingConfig(
                     model_name="nomic-ai/nomic-embed-text-v1.5",
                     dimension=768,
-                    provider=embedding_provider,
+                    # Mode A: sentence-transformers in SUBPROCESS (never in-process)
+                    provider=embedding_provider or "sentence-transformers",
                 ),
                 llm=LLMConfig(),  # No LLM
                 retrieval=RetrievalConfig(
-                    use_cross_encoder=True,
+                    # Mode A: no cross-encoder (saves ~1.5GB PyTorch RAM)
+                    use_cross_encoder=False,
                 ),
                 math=MathConfig(
                     sheaf_contradiction_threshold=0.45,  # 768d threshold
@@ -474,7 +740,8 @@ class SLMConfig:
                 embedding=EmbeddingConfig(
                     model_name="nomic-ai/nomic-embed-text-v1.5",
                     dimension=768,
-                    provider=embedding_provider,
+                    # Mode B: Ollama HTTP API (zero PyTorch in-process)
+                    provider=embedding_provider or "ollama",
                 ),
                 llm=LLMConfig(
                     provider=llm_provider or "ollama",
@@ -482,7 +749,10 @@ class SLMConfig:
                     api_base=llm_api_base or "http://localhost:11434",
                     api_key=llm_api_key or "",
                 ),
-                retrieval=RetrievalConfig(use_cross_encoder=True),
+                retrieval=RetrievalConfig(
+                    # Mode B: no cross-encoder (saves ~1.5GB PyTorch RAM)
+                    use_cross_encoder=False,
+                ),
             )
 
         # Mode C — FULL POWER, UNRESTRICTED
@@ -508,6 +778,7 @@ class SLMConfig:
                 entity_graph=1.3,
                 temporal=1.0,
                 spreading_activation=1.2,  # Phase 3: SA boost in Mode C
+                hopfield=1.0,  # Phase G: Hopfield in Mode C
             ),
             retrieval=RetrievalConfig(
                 use_cross_encoder=True,

package/src/superlocalmemory/core/consolidation_engine.py

@@ -40,6 +40,9 @@ if TYPE_CHECKING:
         BehavioralPatternStore,
         BehavioralTracker,
     )
+    from superlocalmemory.learning.consolidation_quantization_worker import (
+        CCQWorker,
+    )
     from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger(__name__)
@@ -72,6 +75,7 @@ class ConsolidationEngine:
         graph_analyzer: GraphAnalyzer | None = None,
         temporal_validator: TemporalValidator | None = None,
         slm_config: SLMConfig | None = None,
+        ccq_worker: CCQWorker | None = None,
     ) -> None:
         self._db = db
         self._config = config
@@ -81,6 +85,7 @@ class ConsolidationEngine:
         self._graph_analyzer = graph_analyzer
         self._temporal_validator = temporal_validator
         self._slm_config = slm_config
+        self._ccq_worker = ccq_worker
         self._mode = slm_config.mode.value if slm_config else "a"
         self._store_count: int = 0  # For step-count trigger (L7)
 
@@ -119,6 +124,8 @@ class ConsolidationEngine:
                 results["new_associations"] = self._step6_derive_associations(
                     profile_id,
                 )
+                # Step 7: Cognitive Consolidation Quantization (Phase E)
+                results["ccq"] = self._step7_ccq(profile_id)
             results["success"] = True
         except Exception as exc:
             logger.warning(
@@ -463,6 +470,36 @@ class ConsolidationEngine:
                         pass
         return {"summary_facts_linked": linked}
 
+    # ------------------------------------------------------------------
+    # Step 7: Cognitive Consolidation Quantization (Phase E)
+    # ------------------------------------------------------------------
+
+    def _step7_ccq(self, profile_id: str) -> dict[str, Any]:
+        """Run CCQ pipeline after existing 6-step consolidation.
+
+        CCQ is step 7 because it depends on retention data from Phase A
+        and benefits from running after standard consolidation cleanup.
+        """
+        if self._ccq_worker is None:
+            return {"enabled": False}
+
+        if not self._ccq_worker.should_run(
+            self._store_count, is_session_end=False,
+        ):
+            return {"skipped": True, "reason": "trigger not met"}
+
+        try:
+            result = self._ccq_worker.run(profile_id)
+            return {
+                "clusters": result.clusters_processed,
+                "blocks": result.blocks_created,
+                "archived": result.facts_archived,
+                "compression_ratio": result.compression_ratio,
+            }
+        except Exception as exc:
+            logger.warning("CCQ step failed (non-fatal): %s", exc)
+            return {"error": str(exc)}
+
     # ------------------------------------------------------------------
     # Core Memory Block Storage
     # ------------------------------------------------------------------

package/src/superlocalmemory/core/engine.py

@@ -191,6 +191,9 @@ class MemoryEngine:
             behavioral_store=None,
         )
 
+        # V3.3: Check for embedding model migration on mode switch
+        self._check_embedding_migration()
+
         self._initialized = True
         logger.info(
             "MemoryEngine initialized: mode=%s profile=%s",
@@ -320,6 +323,24 @@ class MemoryEngine:
 
     # -- Internal -----------------------------------------------------------
 
+    def _check_embedding_migration(self) -> None:
+        """Detect embedding model change and re-index if needed."""
+        try:
+            from superlocalmemory.storage.embedding_migrator import (
+                check_embedding_migration,
+                run_embedding_migration,
+            )
+            if check_embedding_migration(self._config):
+                count = run_embedding_migration(
+                    self._config, self._db, self._embedder,
+                )
+                if count > 0:
+                    logger.info(
+                        "Embedding migration: %d facts re-embedded", count,
+                    )
+        except Exception as exc:
+            logger.warning("Embedding migration check failed: %s", exc)
+
     def _ensure_init(self) -> None:
         if not self._initialized:
             self.initialize()

package/src/superlocalmemory/core/engine_wiring.py

@@ -64,34 +64,52 @@ def init_embedder(config: SLMConfig) -> Any | None:
 
     Priority order:
     1. Explicit provider in config (ollama / cloud / sentence-transformers)
-    2. Auto-detect: if Ollama has embedding model -> use it
-    3. Fallback to sentence-transformers subprocess
-    4. If nothing works -> None (BM25-only mode)
+    2. Auto-detect: Ollama first (lightweight), then sentence-transformers
+       subprocess (NEVER in-process for Mode A/B)
+    3. If nothing works -> None (BM25-only mode)
+
+    Memory safety: Mode A/B NEVER load sentence-transformers in-process.
+    EmbeddingService uses subprocess isolation — the main process stays
+    at ~60MB and never imports torch.
     """
     from superlocalmemory.core.embeddings import EmbeddingService
+    from superlocalmemory.storage.models import Mode
 
     emb_cfg = config.embedding
     provider = emb_cfg.provider
 
     # --- Explicit ollama provider ---
     if provider == "ollama":
-        return _try_ollama_embedder(emb_cfg)
+        result = _try_ollama_embedder(emb_cfg)
+        if result is not None:
+            return result
+        # Mode B explicitly wants Ollama — if unavailable, fall through
+        # to subprocess (still safe, never in-process)
+        if config.mode == Mode.B:
+            logger.warning(
+                "Ollama unavailable for Mode B. Falling back to "
+                "sentence-transformers subprocess."
+            )
+            return _try_service_embedder(EmbeddingService, emb_cfg)
+        return None
 
     # --- Explicit cloud provider ---
     if provider == "cloud" or emb_cfg.is_cloud:
         return _try_service_embedder(EmbeddingService, emb_cfg)
 
-    # --- Explicit sentence-transformers ---
+    # --- Explicit sentence-transformers (subprocess-isolated) ---
     if provider == "sentence-transformers":
         return _try_service_embedder(EmbeddingService, emb_cfg)
 
-    # --- Auto-detect: try Ollama first (fast path, <1s) ---
+    # --- Auto-detect: try Ollama first (lightweight, <1s) ---
     ollama_emb = _try_ollama_embedder(emb_cfg)
     if ollama_emb is not None:
         logger.info("Auto-detected Ollama embeddings (fast path)")
         return ollama_emb
 
     # --- Fallback: sentence-transformers subprocess ---
+    # EmbeddingService ALWAYS uses subprocess isolation (see embeddings.py).
+    # The main process never imports torch — safe for Mode A/B.
     return _try_service_embedder(EmbeddingService, emb_cfg)
 
 
@@ -358,6 +376,24 @@ def _init_auto_invoker(
 # init_retrieval  (was MemoryEngine._init_retrieval)
 # ---------------------------------------------------------------------------
 
+def _init_hopfield_channel(
+    db: DatabaseManager,
+    vector_store: Any,
+    config: SLMConfig,
+) -> Any | None:
+    """Create HopfieldChannel for Phase G 6th retrieval channel."""
+    if not config.hopfield.enabled:
+        return None
+    try:
+        from superlocalmemory.retrieval.hopfield_channel import HopfieldChannel
+        return HopfieldChannel(
+            db=db, vector_store=vector_store, config=config.hopfield,
+        )
+    except Exception as exc:
+        logger.debug("HopfieldChannel init failed: %s", exc)
+        return None
+
+
 def init_retrieval(
     config: SLMConfig,
     db: DatabaseManager,
@@ -366,7 +402,7 @@ def init_retrieval(
     trust_scorer: Any,
     vector_store: Any = None,
 ) -> Any:
-    """Create the RetrievalEngine with 5 channels. Returns it."""
+    """Create the RetrievalEngine with 6 channels. Returns it."""
     from superlocalmemory.retrieval.engine import RetrievalEngine
     from superlocalmemory.retrieval.semantic_channel import SemanticChannel
     from superlocalmemory.retrieval.bm25_channel import BM25Channel
@@ -394,6 +430,11 @@ def init_retrieval(
     if sa_channel is not None:
         channels["spreading_activation"] = sa_channel
 
+    # Phase G: Register Hopfield as 6th channel
+    hopfield_channel = _init_hopfield_channel(db, vector_store, config)
+    if hopfield_channel is not None:
+        channels["hopfield"] = hopfield_channel
+
     reranker = None
     if config.retrieval.use_cross_encoder:
         reranker = CrossEncoderReranker(config.retrieval.cross_encoder_model)
@@ -401,7 +442,7 @@ def init_retrieval(
     profile_ch = ProfileChannel(db)
     bridge = BridgeDiscovery(db)
 
-    return RetrievalEngine(
+    engine = RetrievalEngine(
         db=db, config=config.retrieval, channels=channels,
         embedder=embedder, reranker=reranker,
         base_weights=config.channel_weights,
@@ -410,6 +451,15 @@ def init_retrieval(
         trust_scorer=trust_scorer,
     )
 
+    # Phase A: Register forgetting filter into the channel registry
+    try:
+        from superlocalmemory.retrieval.forgetting_filter import register_forgetting_filter
+        register_forgetting_filter(engine._registry, db, config.forgetting)
+    except Exception as exc:
+        logger.debug("Forgetting filter registration failed: %s", exc)
+
+    return engine
+
 
 # ---------------------------------------------------------------------------
 # wire_hooks  (was MemoryEngine._wire_hooks)