superlocalmemory 3.8.0 → 3.8.1

package/src/superlocalmemory/core/ingestion_command.py

@@ -24,6 +24,7 @@ from typing import Any, Callable
 from superlocalmemory.storage.database import DatabaseManager
 
 _MATERIALIZATION_LOCKS = tuple(threading.RLock() for _ in range(64))
+_MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS = 10
 
 
 def _materialization_lock(operation_id: str) -> threading.RLock:
@@ -51,6 +52,10 @@ class OperationInProgress(RuntimeError):
     """Another live lease owner is materializing this operation."""
 
 
+class LeaseLost(OperationInProgress):
+    """The materializer no longer owns its durable operation lease."""
+
+
 def _canonical_json(value: Any) -> str:
     return json.dumps(value, sort_keys=True, separators=(",", ":"))
 
@@ -284,12 +289,19 @@ class IngestionOperationRepository:
             self._from_row(row)
             for row in self.db.execute(
                 "SELECT * FROM ingestion_operations "
-                "WHERE (state='queryable' AND created_at <= "
+                "WHERE attempt_count < ? AND ("
+                "(state='queryable' AND created_at <= "
                 "strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ?)) "
                 "OR (state='failed' AND next_retry_at <= ?) "
-                "OR (state='enriching' AND lease_expires_at <= ?) "
+                "OR (state='enriching' AND lease_expires_at <= ?)) "
                 "ORDER BY created_at, rowid LIMIT ?",
-                (grace_modifier, now, now, max(1, int(limit))),
+                (
+                    _MAX_AUTOMATIC_MATERIALIZATION_ATTEMPTS,
+                    grace_modifier,
+                    now,
+                    now,
+                    max(1, int(limit)),
+                ),
             )
         ]
 
@@ -415,6 +427,27 @@ class IngestionOperationRepository:
         )
         return operation
 
+    def renew_enriching_lease(
+        self,
+        operation_id: str,
+        *,
+        owner: str,
+        lease_seconds: float,
+    ) -> bool:
+        """Extend a live lease only while the same owner still holds it."""
+        rows = self.db.execute(
+            "UPDATE ingestion_operations SET lease_expires_at=?, "
+            "updated_at=strftime('%Y-%m-%dT%H:%M:%fZ', 'now') "
+            "WHERE operation_id=? AND state='enriching' AND lease_owner=? "
+            "RETURNING operation_id",
+            (
+                time.time() + max(1.0, float(lease_seconds)),
+                operation_id,
+                owner,
+            ),
+        )
+        return bool(rows)
+
     def finish_enriching(
         self,
         operation_id: str,
@@ -469,6 +502,7 @@ class MaterializationResult:
 
     fact_ids: tuple[str, ...]
     derivation_state: dict[str, bool]
+    last_error: str = ""
 
 
 QueryableWriter = Callable[[IngestionRequest, str], list[str]]
@@ -500,6 +534,47 @@ class IngestionCommand:
         self._lease_seconds = max(1.0, float(lease_seconds))
         self._owner = f"ingestion-worker:{uuid.uuid4().hex}"
 
+    def _run_with_lease_heartbeat(
+        self,
+        operation_id: str,
+        callback: Callable[[], Any],
+    ) -> Any:
+        """Run slow work while periodically renewing its owner-bound lease."""
+        stop = threading.Event()
+        lost = threading.Event()
+        interval = min(30.0, max(0.1, self._lease_seconds / 3.0))
+
+        def heartbeat() -> None:
+            while not stop.wait(interval):
+                try:
+                    renewed = self.repository.renew_enriching_lease(
+                        operation_id,
+                        owner=self._owner,
+                        lease_seconds=self._lease_seconds,
+                    )
+                except sqlite3.Error:
+                    continue
+                if not renewed:
+                    lost.set()
+                    return
+
+        thread = threading.Thread(
+            target=heartbeat,
+            name=f"ingestion-lease-heartbeat:{operation_id[:8]}",
+            daemon=True,
+        )
+        thread.start()
+        try:
+            result = callback()
+        finally:
+            stop.set()
+            thread.join(timeout=max(1.0, interval * 2))
+        if lost.is_set():
+            raise LeaseLost(
+                f"ingestion lease lost for operation {operation_id}"
+            )
+        return result
+
     def submit(self, request: IngestionRequest) -> IngestionOperation:
         operation, _created = self.submit_with_status(request)
         return operation
@@ -548,27 +623,41 @@ class IngestionCommand:
         )
         if enriching.state is IngestionState.COMPLETE:
             return enriching
-        if enriching.final_fact_ids:
+        if (
+            enriching.final_fact_ids
+            and all(enriching.derivation_state.values())
+        ):
             return self._project_and_complete(enriching)
         try:
+            # Materialization is a durable saga, not one long SQLite
+            # transaction. Extractors, embedders, and local model calls can
+            # take minutes on a mature installation; keeping a write
+            # transaction open across that work blocks every interactive
+            # remember/update/delete and makes the dashboard appear dead.
+            #
+            # The materializer commits its relational checkpoints in short
+            # database operations. The operation lease and derivation state
+            # remain the recovery boundary, and only the final state-machine
+            # checkpoint is grouped atomically below.
+            materialized = self._run_with_lease_heartbeat(
+                operation_id,
+                lambda: self._materializer(enriching),
+            )
+            if isinstance(materialized, MaterializationResult):
+                fact_ids = tuple(materialized.fact_ids)
+                derivation_state = dict(materialized.derivation_state)
+                materialization_error = materialized.last_error
+            else:
+                fact_ids = tuple(materialized)
+                derivation_state = {"materializer": True}
+                materialization_error = ""
+            if not fact_ids:
+                raise RuntimeError("materialization produced no final facts")
+            incomplete = sorted(
+                name for name, complete in derivation_state.items()
+                if not complete
+            )
             with self.repository.db.transaction():
-                materialized = self._materializer(enriching)
-                if isinstance(materialized, MaterializationResult):
-                    fact_ids = tuple(materialized.fact_ids)
-                    derivation_state = dict(materialized.derivation_state)
-                else:
-                    fact_ids = tuple(materialized)
-                    derivation_state = {"materializer": True}
-                if not fact_ids:
-                    raise RuntimeError("materialization produced no final facts")
-                incomplete = sorted(
-                    name for name, complete in derivation_state.items()
-                    if not complete
-                )
-                if incomplete:
-                    raise RuntimeError(
-                        "incomplete derivation stages: " + ", ".join(incomplete)
-                    )
                 checkpointed = self.repository.checkpoint_enriching(
                     operation_id,
                     final_fact_ids=fact_ids,
@@ -577,6 +666,21 @@ class IngestionCommand:
                     lease_owner=self._owner,
                     lease_seconds=self._lease_seconds,
                 )
+            if materialization_error or incomplete:
+                error = materialization_error or (
+                    "incomplete derivation stages: " + ", ".join(incomplete)
+                )
+                return self.repository.finish_enriching(
+                    operation_id,
+                    owner=self._owner,
+                    target=IngestionState.FAILED,
+                    final_fact_ids=fact_ids,
+                    derivation_version=self._derivation_version,
+                    derivation_state=derivation_state,
+                    last_error=error,
+                )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation_id,
@@ -598,10 +702,13 @@ class IngestionCommand:
                 owner=self._owner,
                 lease_seconds=self._lease_seconds,
             )
-            projection_state = (
-                dict(self._projector(operation))
-                if self._projector is not None
-                else {}
+            projection_state = self._run_with_lease_heartbeat(
+                operation.operation_id,
+                lambda: (
+                    dict(self._projector(operation))
+                    if self._projector is not None
+                    else {}
+                ),
             )
             combined = {**operation.derivation_state, **projection_state}
             incomplete = sorted(
@@ -619,6 +726,8 @@ class IngestionCommand:
                 derivation_version=self._derivation_version,
                 derivation_state=combined,
             )
+        except LeaseLost:
+            raise
         except Exception as exc:
             return self.repository.finish_enriching(
                 operation.operation_id,

package/src/superlocalmemory/core/injection.py

@@ -128,20 +128,25 @@ def _load_injection_config():
 
 
 def estimate_tokens(text: str) -> int:
-    """chars/4 heuristic. Optional tiktoken if installed (best-effort).
-
-    tiktoken is an optional dependency (pip install superlocalmemory[injection]).
-    Falls back to chars/4 if tiktoken is not installed or raises any error.
-    All exception types (MemoryError, SystemError etc.) are subclasses of
-    Exception in Python ≥ 3.11, so the bare except Exception covers all paths.
+    """Estimate context tokens without loading native code by default.
+
+    The stable default is the deterministic ``chars / 4`` heuristic.  Loading
+    the optional native ``tiktoken`` extension in a long-lived daemon that also
+    hosts torch/scipy workers can terminate the whole process on a native fault;
+    Python exception handling cannot recover from SIGBUS/SIGSEGV.  Users who
+    explicitly need exact OpenAI token counts can opt in with
+    ``SLM_INJECTION_EXACT_TOKENS=1``.
     """
     if not text:
         return 0
+    heuristic = max(1, len(text) // 4)
+    if os.environ.get("SLM_INJECTION_EXACT_TOKENS", "0") != "1":
+        return heuristic
     try:
         import tiktoken
         return len(tiktoken.get_encoding("cl100k_base").encode(text))
     except Exception:
-        return max(1, len(text) // 4)
+        return heuristic
 
 
 def resolve_budget(mode: str, cfg) -> int:

package/src/superlocalmemory/core/maintenance_scheduler.py

@@ -8,7 +8,8 @@ V3.3.13: Periodically triggers Langevin/Ebbinghaus/Sheaf maintenance
 so users don't need to call run_maintenance manually.
 
 Configurable interval via ForgettingConfig.scheduler_interval_minutes.
-Defaults to 30 min. Disabled during benchmarks (no config.forgetting.enabled).
+Defaults to 30 min. Optional forgetting/math work follows
+``config.forgetting.enabled``; tier evaluation and bounded housekeeping do not.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
 License: AGPL-3.0-or-later
@@ -79,12 +80,21 @@ class MaintenanceScheduler:
         if not self._running:
             return
         for profile_id in self._profile_ids():
-            try:
-                from superlocalmemory.core.maintenance import run_maintenance
-                counts = run_maintenance(self._db, self._config, profile_id)
-                logger.info("Scheduled maintenance complete for %s: %s", profile_id, counts)
-            except Exception as exc:
-                logger.warning("Scheduled maintenance failed for %s: %s", profile_id, exc)
+            if self._config.forgetting.enabled:
+                try:
+                    from superlocalmemory.core.maintenance import run_maintenance
+                    counts = run_maintenance(self._db, self._config, profile_id)
+                    logger.info(
+                        "Scheduled maintenance complete for %s: %s",
+                        profile_id,
+                        counts,
+                    )
+                except Exception as exc:
+                    logger.warning(
+                        "Scheduled maintenance failed for %s: %s",
+                        profile_id,
+                        exc,
+                    )
 
             # V3.4.11: Graph pruning (remove orphan edges)
             try:

package/src/superlocalmemory/core/recall_pipeline.py

@@ -632,8 +632,8 @@ def run_recall(
 
     Pipeline: retrieval -> agentic sufficiency (if configured) -> post-recall updates.
 
-    ``fast=True`` skips spreading activation and remote agentic verification,
-    retaining the bounded single-pass retrieval channels.
+    ``fast=True`` skips remote agentic verification while retaining the six
+    local retrieval channels.
     """
     # Pre-operation hooks
     hook_ctx = {
@@ -658,7 +658,9 @@ def run_recall(
             logger.warning("[RECALL-TIMING] %-22s %.0f ms",
                            _label, (_time_t.monotonic() - _t0) * 1000.0)
 
-    extra_disabled = {"spreading_activation"} if fast else None
+    # The interactive path must retain the complete local retrieval contract.
+    # Only agentic verification can invoke an unbounded model round.
+    extra_disabled = None
     response = retrieval_engine.recall(
         query, profile_id, m, limit,
         extra_disabled_channels=extra_disabled,

package/src/superlocalmemory/core/store_pipeline.py

@@ -11,8 +11,10 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 
 from __future__ import annotations
 
-import logging
+import hashlib
 import json
+import logging
+import uuid
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -21,7 +23,9 @@ if TYPE_CHECKING:
     from superlocalmemory.storage.database import DatabaseManager
 
 from superlocalmemory.storage.models import (
-    AtomicFact, FactType, MemoryRecord,
+    AtomicFact,
+    FactType,
+    MemoryRecord,
 )
 
 logger = logging.getLogger(__name__)
@@ -30,6 +34,63 @@ logger = logging.getLogger(__name__)
 _INIT_LANGEVIN_RADIUS = 0.05
 
 
+def _ingestion_effect_id(operation_id: str, *parts: object) -> str:
+    """Return a stable ID for a relational effect owned by one ingestion."""
+    if not operation_id:
+        return uuid.uuid4().hex
+    payload = "\0".join((operation_id, *(str(part) for part in parts)))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
+
+
+def _record_fact_entity_association(
+    db: DatabaseManager,
+    *,
+    operation_id: str,
+    profile_id: str,
+    fact_id: str,
+    entity_id: str,
+) -> None:
+    """Apply one fact/entity count effect in O(1), exactly once."""
+    if not operation_id:
+        db.increment_entity_fact_count(entity_id, profile_id)
+        return
+    with db.transaction():
+        claimed = db.execute(
+            "INSERT INTO fact_entity_associations "
+            "(profile_id,fact_id,entity_id,first_operation_id,count_applied) "
+            "SELECT ?,?,?,?,"
+            "CASE WHEN fact.rowid > repair.target_fact_rowid THEN 1 ELSE 0 END "
+            "FROM canonical_entities AS entity "
+            "JOIN atomic_facts AS fact "
+            "ON fact.fact_id=? AND fact.profile_id=? "
+            "JOIN fact_entity_association_repair_state AS repair "
+            "ON repair.repair_key='historical-backfill' "
+            "WHERE entity.entity_id=? AND entity.profile_id=? "
+            "ON CONFLICT(profile_id,fact_id,entity_id) DO UPDATE SET "
+            "count_applied=excluded.count_applied,"
+            "first_operation_id=excluded.first_operation_id "
+            "WHERE fact_entity_associations.count_applied=0 "
+            "AND excluded.count_applied=1 "
+            "RETURNING count_applied",
+            (
+                profile_id,
+                fact_id,
+                entity_id,
+                operation_id,
+                fact_id,
+                profile_id,
+                entity_id,
+                profile_id,
+            ),
+        )
+        if claimed and int(claimed[0]["count_applied"]) == 1:
+            db.execute(
+                "UPDATE canonical_entities SET fact_count=fact_count+1 "
+                "WHERE entity_id=? AND profile_id=?",
+                (entity_id, profile_id),
+            )
+
+
 def _init_langevin_position(dim: int = 8) -> list[float]:
     """Initialize Langevin position near origin for a new fact.
 
@@ -60,7 +121,7 @@ def enrich_fact(
     temporal_parser: Any,
 ) -> AtomicFact:
     """Enrich fact with embeddings, entities, temporal, emotional data."""
-    from superlocalmemory.encoding.emotional import tag_emotion, emotional_importance_boost
+    from superlocalmemory.encoding.emotional import emotional_importance_boost, tag_emotion
     from superlocalmemory.encoding.signal_inference import infer_signal
 
     embedding = embedder.embed(fact.content) if embedder else None
@@ -189,6 +250,9 @@ def run_store(
     ingestion_source_type: str = "store",
     ingestion_operation_id: str = "",
     derivation_report: dict[str, bool] | None = None,
+    precompleted_derivation_stages: frozenset[str] = frozenset(),
+    materialization_progress: dict[str, Any] | None = None,
+    materialization_checkpoint: Any = None,
 ) -> list[str]:
     """Store content and extract structured facts. Returns fact_ids.
 
@@ -286,10 +350,19 @@ def run_store(
         )
         db.store_memory(record)
 
-    extraction_complete = False
-    consolidation_complete = consolidator is not None
-    canonicalization_complete = entity_resolver is not None
-    graph_complete = graph_builder is not None
+    extraction_complete = "extraction" in precompleted_derivation_stages
+    consolidation_complete = (
+        "consolidation" in precompleted_derivation_stages
+        or consolidator is not None
+    )
+    canonicalization_complete = (
+        "canonicalization" in precompleted_derivation_stages
+        or entity_resolver is not None
+    )
+    graph_complete = (
+        "graph" in precompleted_derivation_stages
+        or graph_builder is not None
+    )
     temporal_complete = True
     provenance_complete = provenance is not None
 
@@ -330,7 +403,6 @@ def run_store(
             and content.strip()
             and len(content.strip()) >= 40
             and len(content.strip().split()) >= _MIN_VERBATIM_WORDS):
-        import uuid
         import re as _re
         _verbatim_text = content.strip()
         # Extract entities using the same regex as fact_extractor
@@ -374,7 +446,6 @@ def run_store(
     # their data should NEVER be silently dropped. The min-length and min-word filters
     # are designed for automatic conversation extraction, not explicit user storage.
     if not facts and content.strip():
-        import uuid
         facts = [AtomicFact(
             fact_id=uuid.uuid4().hex[:16],
             content=content.strip(),
@@ -403,6 +474,22 @@ def run_store(
             temporal_parser=temporal_parser,
         )
 
+        if (
+            materialization_progress is not None
+            and not materialization_progress.get("relational_started", False)
+        ):
+            if materialization_checkpoint is not None:
+                materialization_checkpoint(
+                    "relational_started",
+                    (),
+                    {
+                        "pipeline_started": True,
+                        "relational_started": True,
+                        "pipeline": False,
+                    },
+                )
+            materialization_progress["relational_started"] = True
+
         is_queryable_promotion = fact.fact_id in queryable_ids
         if is_queryable_promotion:
             db.update_fact(fact.fact_id, {
@@ -515,6 +602,8 @@ def run_store(
 
         if fact.fact_id not in stored_ids:
             stored_ids.append(fact.fact_id)
+            if materialization_progress is not None:
+                materialization_progress["fact_ids"] = tuple(stored_ids)
 
         # Dual-write embedding to ANN index + vector store (embed on-demand if
         # a consolidated ADD fact arrived without one). See _upsert_fact_vectors.
@@ -611,12 +700,17 @@ def run_store(
             for eid in fact.canonical_entities:
                 observation_builder.update_profile(eid, fact, profile_id)
 
-        # Increment fact_count for each linked canonical entity (scoped to profile).
+        # The normalized association key makes this O(1) and exactly-once
+        # across retries, crashes, and separate operations that consolidate to
+        # the same fact.
         for eid in fact.canonical_entities:
-            try:
-                db.increment_entity_fact_count(eid, profile_id)
-            except Exception:
-                pass  # Non-critical — entity may have been deleted
+            _record_fact_entity_association(
+                db,
+                operation_id=ingestion_operation_id,
+                profile_id=profile_id,
+                fact_id=fact.fact_id,
+                entity_id=eid,
+            )
         if scene_builder:
             scene_builder.assign_to_scene(fact, profile_id)
 
@@ -627,6 +721,13 @@ def run_store(
             from superlocalmemory.storage.models import TemporalEvent
             for eid in fact.canonical_entities:
                 event = TemporalEvent(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        eid,
+                        "observed",
+                    ),
                     profile_id=profile_id, entity_id=eid,
                     fact_id=fact.fact_id,
                     scope=fact.scope,
@@ -644,8 +745,16 @@ def run_store(
             from superlocalmemory.encoding.foresight import extract_foresight_signals
             from superlocalmemory.storage.models import TemporalEvent as _TE
             foresight_signals = extract_foresight_signals(fact)
-            for sig in foresight_signals:
+            for signal_index, sig in enumerate(foresight_signals):
                 f_event = _TE(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        sig.get("entity_id", ""),
+                        "foresight",
+                        signal_index,
+                    ),
                     profile_id=profile_id,
                     entity_id=sig.get("entity_id", ""),
                     fact_id=fact.fact_id,
@@ -679,11 +788,6 @@ def run_store(
 
     logger.info("Stored %d facts (session=%s)", len(stored_ids), session_id)
 
-    # Post-operation hooks (audit, trust signal, event bus)
-    hook_ctx["fact_ids"] = stored_ids
-    hook_ctx["fact_count"] = len(stored_ids)
-    hooks.run_post("store", hook_ctx)
-
     if derivation_report is not None:
         derivation_report.update({
             "extraction": extraction_complete,
@@ -694,6 +798,35 @@ def run_store(
             "provenance": provenance_complete,
         })
 
+    # Stage observations and emitted fact IDs are recorded before optional
+    # post-hooks.  A hook failure must not erase the durable retry boundary and
+    # cause extraction/consolidation to run again.
+    if materialization_progress is not None:
+        materialization_progress["fact_ids"] = tuple(stored_ids)
+        materialization_progress["relational_complete"] = True
+        materialization_progress["post_hooks"] = False
+    if materialization_checkpoint is not None and stored_ids:
+        materialization_checkpoint(
+            "relational_complete",
+            tuple(stored_ids),
+            {
+                "pipeline_started": True,
+                "relational_started": True,
+                "pipeline": True,
+                "post_hooks": False,
+                "relational": True,
+                **dict(derivation_report or {}),
+            },
+        )
+
+    # Post-operation hooks (audit, trust signal, event bus)
+    hook_ctx["ingestion_operation_id"] = ingestion_operation_id
+    hook_ctx["fact_ids"] = stored_ids
+    hook_ctx["fact_count"] = len(stored_ids)
+    hooks.run_post("store", hook_ctx)
+    if materialization_progress is not None:
+        materialization_progress["post_hooks"] = True
+
     # Phase 5: Step-count trigger for lightweight consolidation (L7)
     if consolidation_engine is not None:
         try:

package/src/superlocalmemory/hooks/hook_handlers.py

@@ -26,6 +26,8 @@ import urllib.error
 import urllib.request
 from pathlib import Path
 
+from superlocalmemory import __version__
+
 # ---------------------------------------------------------------------------
 # Cross-platform temp paths
 # ---------------------------------------------------------------------------
@@ -221,7 +223,10 @@ def _codex_mcp_session_init(project_dir: str, payload: dict) -> dict:
         proc.stdin.write(json.dumps({
             "jsonrpc": "2.0", "id": 1, "method": "initialize",
             "params": {"protocolVersion": "2024-11-05", "capabilities": {},
-                       "clientInfo": {"name": "superlocalmemory-codex-hook", "version": "3.7"}},
+                       "clientInfo": {
+                           "name": "superlocalmemory-codex-hook",
+                           "version": __version__,
+                       }},
         }) + "\n")
         proc.stdin.flush()
         proc.stdout.readline()