superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/core/store_pipeline.py

@@ -11,8 +11,10 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 
 from __future__ import annotations
 
-import logging
+import hashlib
 import json
+import logging
+import uuid
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -21,7 +23,9 @@ if TYPE_CHECKING:
     from superlocalmemory.storage.database import DatabaseManager
 
 from superlocalmemory.storage.models import (
-    AtomicFact, FactType, MemoryRecord,
+    AtomicFact,
+    FactType,
+    MemoryRecord,
 )
 
 logger = logging.getLogger(__name__)
@@ -30,6 +34,63 @@ logger = logging.getLogger(__name__)
 _INIT_LANGEVIN_RADIUS = 0.05
 
 
+def _ingestion_effect_id(operation_id: str, *parts: object) -> str:
+    """Return a stable ID for a relational effect owned by one ingestion."""
+    if not operation_id:
+        return uuid.uuid4().hex
+    payload = "\0".join((operation_id, *(str(part) for part in parts)))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
+
+
+def _record_fact_entity_association(
+    db: DatabaseManager,
+    *,
+    operation_id: str,
+    profile_id: str,
+    fact_id: str,
+    entity_id: str,
+) -> None:
+    """Apply one fact/entity count effect in O(1), exactly once."""
+    if not operation_id:
+        db.increment_entity_fact_count(entity_id, profile_id)
+        return
+    with db.transaction():
+        claimed = db.execute(
+            "INSERT INTO fact_entity_associations "
+            "(profile_id,fact_id,entity_id,first_operation_id,count_applied) "
+            "SELECT ?,?,?,?,"
+            "CASE WHEN fact.rowid > repair.target_fact_rowid THEN 1 ELSE 0 END "
+            "FROM canonical_entities AS entity "
+            "JOIN atomic_facts AS fact "
+            "ON fact.fact_id=? AND fact.profile_id=? "
+            "JOIN fact_entity_association_repair_state AS repair "
+            "ON repair.repair_key='historical-backfill' "
+            "WHERE entity.entity_id=? AND entity.profile_id=? "
+            "ON CONFLICT(profile_id,fact_id,entity_id) DO UPDATE SET "
+            "count_applied=excluded.count_applied,"
+            "first_operation_id=excluded.first_operation_id "
+            "WHERE fact_entity_associations.count_applied=0 "
+            "AND excluded.count_applied=1 "
+            "RETURNING count_applied",
+            (
+                profile_id,
+                fact_id,
+                entity_id,
+                operation_id,
+                fact_id,
+                profile_id,
+                entity_id,
+                profile_id,
+            ),
+        )
+        if claimed and int(claimed[0]["count_applied"]) == 1:
+            db.execute(
+                "UPDATE canonical_entities SET fact_count=fact_count+1 "
+                "WHERE entity_id=? AND profile_id=?",
+                (entity_id, profile_id),
+            )
+
+
 def _init_langevin_position(dim: int = 8) -> list[float]:
     """Initialize Langevin position near origin for a new fact.
 
@@ -60,7 +121,7 @@ def enrich_fact(
     temporal_parser: Any,
 ) -> AtomicFact:
     """Enrich fact with embeddings, entities, temporal, emotional data."""
-    from superlocalmemory.encoding.emotional import tag_emotion, emotional_importance_boost
+    from superlocalmemory.encoding.emotional import emotional_importance_boost, tag_emotion
     from superlocalmemory.encoding.signal_inference import infer_signal
 
     embedding = embedder.embed(fact.content) if embedder else None
@@ -189,6 +250,9 @@ def run_store(
     ingestion_source_type: str = "store",
     ingestion_operation_id: str = "",
     derivation_report: dict[str, bool] | None = None,
+    precompleted_derivation_stages: frozenset[str] = frozenset(),
+    materialization_progress: dict[str, Any] | None = None,
+    materialization_checkpoint: Any = None,
 ) -> list[str]:
     """Store content and extract structured facts. Returns fact_ids.
 
@@ -208,21 +272,33 @@ def run_store(
     if not pre_authorized:
         hooks.run_pre("store", hook_ctx)
 
-    if entropy_gate and not entropy_gate.should_pass(content):
+    # Admission gates apply to FRESH submissions only. A materialization pass
+    # re-runs this pipeline for content whose queryable projection was already
+    # committed at submit (``queryable_fact_ids`` is set). Re-applying admission
+    # here — the entropy near-duplicate gate especially — would discard that
+    # already-committed fact (return []) and wedge the operation in an endless
+    # materialize retry loop ("materialization produced no final facts"). The
+    # near-duplicate verdict is expected at materialize: the submitted
+    # projection itself is in the gate's window. Admission was already decided
+    # at submit (store_fast enforces the same gates), so skip it here.
+    is_materialization = bool(queryable_fact_ids)
+
+    if entropy_gate and not is_materialization and not entropy_gate.should_pass(content):
         return []
 
     # v3.5.0: store-side quality gate (H3). Reject prompt-template leakage,
     # empty placeholders, and other non-memory content BEFORE it enters the
     # DB. Uses the shared is_low_quality from core/injection so both store
     # AND injection filter by identical rules. Saves DB IO + recall pollution.
-    try:
-        from superlocalmemory.core.injection import is_low_quality
-        if is_low_quality(content):
-            logger.debug("Store rejected (low-quality content): %s...",
-                         content[:80].replace("\n", " "))
-            return []
-    except Exception:
-        pass  # Best-effort gate; store succeeds if import fails
+    if not is_materialization:
+        try:
+            from superlocalmemory.core.injection import is_low_quality
+            if is_low_quality(content):
+                logger.debug("Store rejected (low-quality content): %s...",
+                             content[:80].replace("\n", " "))
+                return []
+        except Exception:
+            pass  # Best-effort gate; store succeeds if import fails
 
     from superlocalmemory.encoding.temporal_parser import TemporalParser
     parser = temporal_parser or TemporalParser()
@@ -274,10 +350,19 @@ def run_store(
         )
         db.store_memory(record)
 
-    extraction_complete = False
-    consolidation_complete = consolidator is not None
-    canonicalization_complete = entity_resolver is not None
-    graph_complete = graph_builder is not None
+    extraction_complete = "extraction" in precompleted_derivation_stages
+    consolidation_complete = (
+        "consolidation" in precompleted_derivation_stages
+        or consolidator is not None
+    )
+    canonicalization_complete = (
+        "canonicalization" in precompleted_derivation_stages
+        or entity_resolver is not None
+    )
+    graph_complete = (
+        "graph" in precompleted_derivation_stages
+        or graph_builder is not None
+    )
     temporal_complete = True
     provenance_complete = provenance is not None
 
@@ -318,7 +403,6 @@ def run_store(
             and content.strip()
             and len(content.strip()) >= 40
             and len(content.strip().split()) >= _MIN_VERBATIM_WORDS):
-        import uuid
         import re as _re
         _verbatim_text = content.strip()
         # Extract entities using the same regex as fact_extractor
@@ -362,7 +446,6 @@ def run_store(
     # their data should NEVER be silently dropped. The min-length and min-word filters
     # are designed for automatic conversation extraction, not explicit user storage.
     if not facts and content.strip():
-        import uuid
         facts = [AtomicFact(
             fact_id=uuid.uuid4().hex[:16],
             content=content.strip(),
@@ -391,6 +474,22 @@ def run_store(
             temporal_parser=temporal_parser,
         )
 
+        if (
+            materialization_progress is not None
+            and not materialization_progress.get("relational_started", False)
+        ):
+            if materialization_checkpoint is not None:
+                materialization_checkpoint(
+                    "relational_started",
+                    (),
+                    {
+                        "pipeline_started": True,
+                        "relational_started": True,
+                        "pipeline": False,
+                    },
+                )
+            materialization_progress["relational_started"] = True
+
         is_queryable_promotion = fact.fact_id in queryable_ids
         if is_queryable_promotion:
             db.update_fact(fact.fact_id, {
@@ -503,6 +602,8 @@ def run_store(
 
         if fact.fact_id not in stored_ids:
             stored_ids.append(fact.fact_id)
+            if materialization_progress is not None:
+                materialization_progress["fact_ids"] = tuple(stored_ids)
 
         # Dual-write embedding to ANN index + vector store (embed on-demand if
         # a consolidated ADD fact arrived without one). See _upsert_fact_vectors.
@@ -583,16 +684,33 @@ def run_store(
                     fact.fact_id, exc,
                 )
 
+        # Phase 4b: fact-augmented key expansion (T3b). Index entity aliases /
+        # canonical names as BM25 alt-keys so paraphrased queries match. Mode A
+        # (entity graph) is zero-LLM and safe on the hot store path; LLM
+        # paraphrase enrichment (Mode B/C) is left to background consolidation.
+        try:
+            from superlocalmemory.core.key_expander import KeyExpander
+            _alt_keys = KeyExpander(db).expand(fact, profile_id, mode="a")
+            if _alt_keys:
+                db.upsert_fact_expansion(fact.fact_id, _alt_keys)
+        except Exception as exc:
+            logger.debug("Key expansion skipped for %s: %s", fact.fact_id, exc)
+
         if observation_builder:
             for eid in fact.canonical_entities:
                 observation_builder.update_profile(eid, fact, profile_id)
 
-        # Increment fact_count for each linked canonical entity
+        # The normalized association key makes this O(1) and exactly-once
+        # across retries, crashes, and separate operations that consolidate to
+        # the same fact.
         for eid in fact.canonical_entities:
-            try:
-                db.increment_entity_fact_count(eid)
-            except Exception:
-                pass  # Non-critical — entity may have been deleted
+            _record_fact_entity_association(
+                db,
+                operation_id=ingestion_operation_id,
+                profile_id=profile_id,
+                fact_id=fact.fact_id,
+                entity_id=eid,
+            )
         if scene_builder:
             scene_builder.assign_to_scene(fact, profile_id)
 
@@ -603,6 +721,13 @@ def run_store(
             from superlocalmemory.storage.models import TemporalEvent
             for eid in fact.canonical_entities:
                 event = TemporalEvent(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        eid,
+                        "observed",
+                    ),
                     profile_id=profile_id, entity_id=eid,
                     fact_id=fact.fact_id,
                     scope=fact.scope,
@@ -620,8 +745,16 @@ def run_store(
             from superlocalmemory.encoding.foresight import extract_foresight_signals
             from superlocalmemory.storage.models import TemporalEvent as _TE
             foresight_signals = extract_foresight_signals(fact)
-            for sig in foresight_signals:
+            for signal_index, sig in enumerate(foresight_signals):
                 f_event = _TE(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        sig.get("entity_id", ""),
+                        "foresight",
+                        signal_index,
+                    ),
                     profile_id=profile_id,
                     entity_id=sig.get("entity_id", ""),
                     fact_id=fact.fact_id,
@@ -655,11 +788,6 @@ def run_store(
 
     logger.info("Stored %d facts (session=%s)", len(stored_ids), session_id)
 
-    # Post-operation hooks (audit, trust signal, event bus)
-    hook_ctx["fact_ids"] = stored_ids
-    hook_ctx["fact_count"] = len(stored_ids)
-    hooks.run_post("store", hook_ctx)
-
     if derivation_report is not None:
         derivation_report.update({
             "extraction": extraction_complete,
@@ -670,6 +798,35 @@ def run_store(
             "provenance": provenance_complete,
         })
 
+    # Stage observations and emitted fact IDs are recorded before optional
+    # post-hooks.  A hook failure must not erase the durable retry boundary and
+    # cause extraction/consolidation to run again.
+    if materialization_progress is not None:
+        materialization_progress["fact_ids"] = tuple(stored_ids)
+        materialization_progress["relational_complete"] = True
+        materialization_progress["post_hooks"] = False
+    if materialization_checkpoint is not None and stored_ids:
+        materialization_checkpoint(
+            "relational_complete",
+            tuple(stored_ids),
+            {
+                "pipeline_started": True,
+                "relational_started": True,
+                "pipeline": True,
+                "post_hooks": False,
+                "relational": True,
+                **dict(derivation_report or {}),
+            },
+        )
+
+    # Post-operation hooks (audit, trust signal, event bus)
+    hook_ctx["ingestion_operation_id"] = ingestion_operation_id
+    hook_ctx["fact_ids"] = stored_ids
+    hook_ctx["fact_count"] = len(stored_ids)
+    hooks.run_post("store", hook_ctx)
+    if materialization_progress is not None:
+        materialization_progress["post_hooks"] = True
+
     # Phase 5: Step-count trigger for lightweight consolidation (L7)
     if consolidation_engine is not None:
         try:

package/src/superlocalmemory/core/worker_pool.py

@@ -70,6 +70,7 @@ class WorkerPool:
         fast: bool = False,
         include_global: bool | None = None,
         include_shared: bool | None = None,
+        window: str | None = None,
     ) -> dict:
         """Run recall in worker subprocess. Returns result dict.
 
@@ -91,6 +92,8 @@ class WorkerPool:
             msg["include_global"] = bool(include_global)
         if include_shared is not None:
             msg["include_shared"] = bool(include_shared)
+        if window:
+            msg["window"] = window
         return self._send(msg)
 
     def store(self, content: str, metadata: dict | None = None) -> dict:
@@ -291,15 +294,27 @@ class WorkerPool:
             self._idle_timer.cancel()
             self._idle_timer = None
         if self._proc is not None:
-            pid = self._proc.pid
+            proc = self._proc
+            pid = proc.pid
             try:
-                self._proc.stdin.write('{"cmd":"quit"}\n')
-                self._proc.stdin.flush()
-                self._proc.wait(timeout=3)
+                proc.stdin.write('{"cmd":"quit"}\n')
+                proc.stdin.flush()
+                proc.wait(timeout=3)
             except Exception:
                 try:
-                    self._proc.kill()
-                    self._proc.wait(timeout=2)
+                    proc.kill()
+                    proc.wait(timeout=2)
+                except Exception:
+                    pass
+            # L-CONC-1: deterministically close the pipe fds rather than leaving
+            # them to GC. This releases the OS handles immediately and unblocks
+            # any orphaned _readline_with_timeout reader thread (its readline
+            # returns/raises on the closed pipe), so repeated request timeouts
+            # cannot accumulate reader threads or file descriptors. Cross-platform.
+            for stream in (proc.stdin, proc.stdout, proc.stderr):
+                try:
+                    if stream is not None:
+                        stream.close()
                 except Exception:
                     pass
             self._proc = None

package/src/superlocalmemory/encoding/entity_reflexion.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Reflexion self-review over LLM-extracted entities (Wave Q1).
+
+After primary LLM extraction (Mode B/C only), one bounded self-review pass
+audits the extracted entities against the source text and:
+
+  - drops hallucinated entities (not supported by the source), and
+  - adds clearly-missed entities (whose exact text appears in the source).
+
+This closes the two distinct extraction failure modes the market leaders
+separate: coverage (missed) and grounding (hallucinated). Coverage is the
+"gleaning" idea (re-present what was extracted, ask for misses); grounding is
+the "reflexion" idea (verify each extracted entity is real).
+
+Design guarantees:
+  - Mode A never reaches here — the caller gates on Mode B/C + LLM available.
+  - Fail-open: any error, unavailable LLM, or unparseable output returns the
+    input facts unchanged. This module NEVER raises.
+  - Immutable: corrected facts are new objects; inputs are not mutated.
+  - Bounded: one LLM call, capped facts reviewed, capped entities per fact.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import replace
+from typing import Any
+
+from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+_MAX_ENTITIES_PER_FACT = 12
+_MIN_ENTITY_LEN = 2
+
+_SYSTEM = (
+    "You are a precise information-extraction auditor. You verify that "
+    "entities extracted from text are actually supported by that text, and "
+    "you catch clearly-named entities that were missed. You never invent "
+    "entities. You reply with JSON only."
+)
+
+
+class EntityReflexion:
+    """Bounded, fail-open reflexion pass over extracted entities."""
+
+    def __init__(
+        self,
+        llm: Any,
+        max_facts: int = 8,
+        max_tokens: int = 512,
+    ) -> None:
+        self._llm = llm
+        self._max_facts = max(1, int(max_facts))
+        self._max_tokens = max(64, int(max_tokens))
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def refine(
+        self, source_text: str, facts: list[AtomicFact],
+    ) -> list[AtomicFact]:
+        """Return facts with entity lists corrected, or unchanged on failure."""
+        if not self._llm or not facts or not source_text or not source_text.strip():
+            return facts
+        try:
+            is_avail = getattr(self._llm, "is_available", None)
+            if callable(is_avail) and not is_avail():
+                return facts
+        except Exception:
+            return facts
+
+        subject = facts[: self._max_facts]
+        try:
+            raw = self._invoke(source_text, subject)
+        except Exception as exc:
+            logger.debug("EntityReflexion LLM call failed: %s", exc)
+            return facts
+
+        corrections = self._parse(raw)
+        if not corrections:
+            return facts
+        try:
+            return self._apply(facts, corrections, source_text)
+        except Exception as exc:
+            logger.debug("EntityReflexion apply failed: %s", exc)
+            return facts
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _invoke(self, source_text: str, subject: list[AtomicFact]) -> str:
+        lines = []
+        for i, f in enumerate(subject):
+            lines.append(f'{i}: "{f.content}" entities={list(f.entities)}')
+        facts_block = "\n".join(lines)
+        prompt = (
+            "Audit the extracted entities for each fact against the SOURCE "
+            "text below.\n"
+            "For each fact, report:\n"
+            "- drop: entities in the fact's list that are NOT supported by the "
+            "source (hallucinated).\n"
+            "- add: named entities clearly present in the source but missing "
+            "from the fact's list. Only add entities whose exact text appears "
+            "in the source.\n\n"
+            f"--- SOURCE ---\n{source_text}\n--- END ---\n\n"
+            f"--- FACTS ---\n{facts_block}\n--- END ---\n\n"
+            'Respond with ONLY a JSON array: '
+            '[{"index": <int>, "drop": [..], "add": [..]}]. '
+            "Use [] for an empty list. Omit facts that need no change."
+        )
+        out = self._llm.generate(
+            prompt=prompt,
+            system=_SYSTEM,
+            temperature=0.0,
+            max_tokens=self._max_tokens,
+        )
+        return out if isinstance(out, str) else str(out or "")
+
+    @staticmethod
+    def _parse(raw: str) -> list[dict]:
+        if not raw or not raw.strip():
+            return []
+        match = re.search(r"\[.*\]", raw, re.DOTALL)
+        if not match:
+            return []
+        try:
+            data = json.loads(match.group())
+        except (json.JSONDecodeError, ValueError):
+            return []
+        if not isinstance(data, list):
+            return []
+        return [d for d in data if isinstance(d, dict)]
+
+    def _apply(
+        self,
+        facts: list[AtomicFact],
+        corrections: list[dict],
+        source_text: str,
+    ) -> list[AtomicFact]:
+        by_index: dict[int, dict] = {}
+        for c in corrections:
+            idx = c.get("index")
+            if isinstance(idx, int) and 0 <= idx < self._max_facts:
+                by_index[idx] = c
+
+        src_low = source_text.lower()
+        out: list[AtomicFact] = []
+        for i, fact in enumerate(facts):
+            c = by_index.get(i)
+            if c is None:
+                out.append(fact)
+                continue
+
+            ents = list(fact.entities)
+            drop = {
+                str(d).strip().lower()
+                for d in _as_list(c.get("drop"))
+                if str(d).strip()
+            }
+            if drop:
+                ents = [e for e in ents if e.strip().lower() not in drop]
+
+            seen = {e.strip().lower() for e in ents}
+            for a in _as_list(c.get("add")):
+                cand = str(a).strip()
+                low = cand.lower()
+                if len(cand) < _MIN_ENTITY_LEN or low in seen:
+                    continue
+                # Grounding guard: only add entities literally present in source.
+                if low not in src_low:
+                    continue
+                ents.append(cand)
+                seen.add(low)
+                if len(ents) >= _MAX_ENTITIES_PER_FACT:
+                    break
+
+            if ents != list(fact.entities):
+                out.append(replace(fact, entities=ents))
+            else:
+                out.append(fact)
+        return out
+
+
+def _as_list(value: Any) -> list:
+    if isinstance(value, list):
+        return value
+    if value is None:
+        return []
+    return [value]