superlocalmemory 3.7.7 → 3.8.0

package/src/superlocalmemory/encoding/entity_reflexion.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Reflexion self-review over LLM-extracted entities (Wave Q1).
+
+After primary LLM extraction (Mode B/C only), one bounded self-review pass
+audits the extracted entities against the source text and:
+
+  - drops hallucinated entities (not supported by the source), and
+  - adds clearly-missed entities (whose exact text appears in the source).
+
+This closes the two distinct extraction failure modes the market leaders
+separate: coverage (missed) and grounding (hallucinated). Coverage is the
+"gleaning" idea (re-present what was extracted, ask for misses); grounding is
+the "reflexion" idea (verify each extracted entity is real).
+
+Design guarantees:
+  - Mode A never reaches here — the caller gates on Mode B/C + LLM available.
+  - Fail-open: any error, unavailable LLM, or unparseable output returns the
+    input facts unchanged. This module NEVER raises.
+  - Immutable: corrected facts are new objects; inputs are not mutated.
+  - Bounded: one LLM call, capped facts reviewed, capped entities per fact.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: AGPL-3.0-or-later
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from dataclasses import replace
+from typing import Any
+
+from superlocalmemory.storage.models import AtomicFact
+
+logger = logging.getLogger(__name__)
+
+_MAX_ENTITIES_PER_FACT = 12
+_MIN_ENTITY_LEN = 2
+
+_SYSTEM = (
+    "You are a precise information-extraction auditor. You verify that "
+    "entities extracted from text are actually supported by that text, and "
+    "you catch clearly-named entities that were missed. You never invent "
+    "entities. You reply with JSON only."
+)
+
+
+class EntityReflexion:
+    """Bounded, fail-open reflexion pass over extracted entities."""
+
+    def __init__(
+        self,
+        llm: Any,
+        max_facts: int = 8,
+        max_tokens: int = 512,
+    ) -> None:
+        self._llm = llm
+        self._max_facts = max(1, int(max_facts))
+        self._max_tokens = max(64, int(max_tokens))
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def refine(
+        self, source_text: str, facts: list[AtomicFact],
+    ) -> list[AtomicFact]:
+        """Return facts with entity lists corrected, or unchanged on failure."""
+        if not self._llm or not facts or not source_text or not source_text.strip():
+            return facts
+        try:
+            is_avail = getattr(self._llm, "is_available", None)
+            if callable(is_avail) and not is_avail():
+                return facts
+        except Exception:
+            return facts
+
+        subject = facts[: self._max_facts]
+        try:
+            raw = self._invoke(source_text, subject)
+        except Exception as exc:
+            logger.debug("EntityReflexion LLM call failed: %s", exc)
+            return facts
+
+        corrections = self._parse(raw)
+        if not corrections:
+            return facts
+        try:
+            return self._apply(facts, corrections, source_text)
+        except Exception as exc:
+            logger.debug("EntityReflexion apply failed: %s", exc)
+            return facts
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _invoke(self, source_text: str, subject: list[AtomicFact]) -> str:
+        lines = []
+        for i, f in enumerate(subject):
+            lines.append(f'{i}: "{f.content}" entities={list(f.entities)}')
+        facts_block = "\n".join(lines)
+        prompt = (
+            "Audit the extracted entities for each fact against the SOURCE "
+            "text below.\n"
+            "For each fact, report:\n"
+            "- drop: entities in the fact's list that are NOT supported by the "
+            "source (hallucinated).\n"
+            "- add: named entities clearly present in the source but missing "
+            "from the fact's list. Only add entities whose exact text appears "
+            "in the source.\n\n"
+            f"--- SOURCE ---\n{source_text}\n--- END ---\n\n"
+            f"--- FACTS ---\n{facts_block}\n--- END ---\n\n"
+            'Respond with ONLY a JSON array: '
+            '[{"index": <int>, "drop": [..], "add": [..]}]. '
+            "Use [] for an empty list. Omit facts that need no change."
+        )
+        out = self._llm.generate(
+            prompt=prompt,
+            system=_SYSTEM,
+            temperature=0.0,
+            max_tokens=self._max_tokens,
+        )
+        return out if isinstance(out, str) else str(out or "")
+
+    @staticmethod
+    def _parse(raw: str) -> list[dict]:
+        if not raw or not raw.strip():
+            return []
+        match = re.search(r"\[.*\]", raw, re.DOTALL)
+        if not match:
+            return []
+        try:
+            data = json.loads(match.group())
+        except (json.JSONDecodeError, ValueError):
+            return []
+        if not isinstance(data, list):
+            return []
+        return [d for d in data if isinstance(d, dict)]
+
+    def _apply(
+        self,
+        facts: list[AtomicFact],
+        corrections: list[dict],
+        source_text: str,
+    ) -> list[AtomicFact]:
+        by_index: dict[int, dict] = {}
+        for c in corrections:
+            idx = c.get("index")
+            if isinstance(idx, int) and 0 <= idx < self._max_facts:
+                by_index[idx] = c
+
+        src_low = source_text.lower()
+        out: list[AtomicFact] = []
+        for i, fact in enumerate(facts):
+            c = by_index.get(i)
+            if c is None:
+                out.append(fact)
+                continue
+
+            ents = list(fact.entities)
+            drop = {
+                str(d).strip().lower()
+                for d in _as_list(c.get("drop"))
+                if str(d).strip()
+            }
+            if drop:
+                ents = [e for e in ents if e.strip().lower() not in drop]
+
+            seen = {e.strip().lower() for e in ents}
+            for a in _as_list(c.get("add")):
+                cand = str(a).strip()
+                low = cand.lower()
+                if len(cand) < _MIN_ENTITY_LEN or low in seen:
+                    continue
+                # Grounding guard: only add entities literally present in source.
+                if low not in src_low:
+                    continue
+                ents.append(cand)
+                seen.add(low)
+                if len(ents) >= _MAX_ENTITIES_PER_FACT:
+                    break
+
+            if ents != list(fact.entities):
+                out.append(replace(fact, entities=ents))
+            else:
+                out.append(fact)
+        return out
+
+
+def _as_list(value: Any) -> list:
+    if isinstance(value, list):
+        return value
+    if value is None:
+        return []
+    return [value]

package/src/superlocalmemory/encoding/entity_resolver.py

@@ -283,22 +283,22 @@ class EntityResolver:
             entity = self._db.get_entity_by_name(name, profile_id)
             if entity is not None:
                 resolution[raw] = entity.entity_id
-                self._touch_last_seen(entity.entity_id)
+                self._touch_last_seen(entity.entity_id, profile_id)
                 continue
 
             # Tier b: alias match (case-insensitive, indexed)
             entity_id = self._alias_lookup(name, profile_id)
             if entity_id is not None:
                 resolution[raw] = entity_id
-                self._touch_last_seen(entity_id)
+                self._touch_last_seen(entity_id, profile_id)
                 continue
 
             # Tier c: fuzzy match via Jaro-Winkler
             match_id, score = self._fuzzy_match(name, profile_id)
             if match_id is not None and score >= JARO_WINKLER_AUTO_MERGE:
                 resolution[raw] = match_id
-                self._persist_alias(match_id, name, score, "jaro_winkler")
-                self._touch_last_seen(match_id)
+                self._persist_alias(match_id, name, score, "jaro_winkler", profile_id)
+                self._touch_last_seen(match_id, profile_id)
                 continue
 
             # Candidate zone (0.70–0.85): queue for LLM in Mode B/C
@@ -384,7 +384,7 @@ class EntityResolver:
         and removes the merged entity record.
         """
         # Move aliases from merge -> keep
-        aliases = self._db.get_aliases_for_entity(entity_id_merge)
+        aliases = self._db.get_aliases_for_entity(entity_id_merge, profile_id)
         for alias in aliases:
             new_alias = EntityAlias(
                 alias_id=_new_id(),
@@ -393,7 +393,7 @@ class EntityResolver:
                 confidence=alias.confidence,
                 source=f"merge_from:{entity_id_merge}",
             )
-            self._db.store_alias(new_alias)
+            self._db.store_alias(new_alias, profile_id)
 
         # Also add the merged entity's canonical name as an alias of keep
         merged = self._db.get_entity_by_name("", "")  # placeholder
@@ -403,7 +403,7 @@ class EntityResolver:
         )
         if rows:
             merged_name = str(dict(rows[0])["canonical_name"])
-            self._persist_alias(entity_id_keep, merged_name, 1.0, "merge")
+            self._persist_alias(entity_id_keep, merged_name, 1.0, "merge", profile_id)
 
         # Update atomic_facts: replace entity_id_merge with entity_id_keep
         # in canonical_entities_json column
@@ -435,14 +435,14 @@ class EntityResolver:
             except (json.JSONDecodeError, TypeError):
                 continue
 
-        # Delete the merged entity
+        # Delete the merged entity (tenant-scoped)
         self._db.execute(
-            "DELETE FROM entity_aliases WHERE entity_id = ?",
-            (entity_id_merge,),
+            "DELETE FROM entity_aliases WHERE entity_id = ? AND profile_id = ?",
+            (entity_id_merge, profile_id),
         )
         self._db.execute(
-            "DELETE FROM canonical_entities WHERE entity_id = ?",
-            (entity_id_merge,),
+            "DELETE FROM canonical_entities WHERE entity_id = ? AND profile_id = ?",
+            (entity_id_merge, profile_id),
         )
         logger.info(
             "Merged entity %s into %s (profile=%s)",
@@ -526,7 +526,7 @@ class EntityResolver:
         self._db.store_entity(entity)
 
         # Store name as its own alias for uniform lookup
-        self._persist_alias(entity.entity_id, name, 1.0, "canonical")
+        self._persist_alias(entity.entity_id, name, 1.0, "canonical", profile_id)
 
         logger.debug(
             "Created entity '%s' [%s] (type=%s, profile=%s)",
@@ -540,13 +540,16 @@ class EntityResolver:
         alias_text: str,
         confidence: float,
         source: str,
+        profile_id: str,
     ) -> None:
-        """Store an alias, skipping duplicates."""
-        # Check if alias already exists for this entity
+        """Store an alias under a profile, skipping duplicates within it."""
+        # Check if alias already exists for this entity IN THIS PROFILE. The
+        # dedup must be profile-scoped or one profile's alias suppresses
+        # another profile's identical alias for a same-id entity.
         existing = self._db.execute(
             "SELECT alias_id FROM entity_aliases "
-            "WHERE entity_id = ? AND LOWER(alias) = LOWER(?)",
-            (entity_id, alias_text),
+            "WHERE entity_id = ? AND profile_id = ? AND LOWER(alias) = LOWER(?)",
+            (entity_id, profile_id, alias_text),
         )
         if existing:
             return
@@ -557,13 +560,20 @@ class EntityResolver:
             confidence=confidence,
             source=source,
         )
-        self._db.store_alias(alias)
+        self._db.store_alias(alias, profile_id)
 
-    def _touch_last_seen(self, entity_id: str) -> None:
-        """Update last_seen timestamp on a canonical entity."""
+    def _touch_last_seen(self, entity_id: str, profile_id: str = "default") -> None:
+        """Update last_seen timestamp on a canonical entity scoped to profile.
+
+        L-01 fix: the original query had no profile_id guard. Since entity_ids are
+        UUIDs today the blast radius is theoretical, but the guard is required for
+        defense-in-depth against future import/sharing features that could introduce
+        UUID collisions across profiles.
+        """
         self._db.execute(
-            "UPDATE canonical_entities SET last_seen = ? WHERE entity_id = ?",
-            (_now(), entity_id),
+            "UPDATE canonical_entities SET last_seen = ? "
+            "WHERE entity_id = ? AND profile_id = ?",
+            (_now(), entity_id, profile_id),
         )
 
     # -- Internal: LLM disambiguation (Mode B/C) ---------------------------
@@ -621,9 +631,9 @@ class EntityResolver:
                     entity_id = known[name_str]
                     resolved[mention_str] = entity_id
                     self._persist_alias(
-                        entity_id, mention_str, 0.9, "llm",
+                        entity_id, mention_str, 0.9, "llm", profile_id,
                     )
-                    self._touch_last_seen(entity_id)
+                    self._touch_last_seen(entity_id, profile_id)
                 # If LLM says it's itself, leave for caller to create
             return resolved
 

package/src/superlocalmemory/encoding/fact_extractor.py

@@ -685,11 +685,36 @@ class FactExtractor:
                 temperature=0.0,
                 max_tokens=1024,
             )
-            return self._parse_llm_response(raw, session_id, session_date)
+            facts = self._parse_llm_response(raw, session_id, session_date)
+            return self._reflexion_refine(conversation_text, facts)
         except Exception as exc:
             logger.warning("LLM fact extraction failed: %s", exc)
             return []
 
+    def _reflexion_refine(
+        self, source_text: str, facts: list[AtomicFact],
+    ) -> list[AtomicFact]:
+        """Wave Q1: bounded, fail-open entity self-review (Mode B/C only).
+
+        Reached only from the LLM extraction path, so Mode A never runs this.
+        Any failure returns the facts unchanged.
+        """
+        if not facts or not getattr(
+            self._config, "enable_entity_reflexion", True,
+        ):
+            return facts
+        try:
+            from superlocalmemory.encoding.entity_reflexion import EntityReflexion
+
+            reflexion = EntityReflexion(
+                self._llm,
+                max_facts=getattr(self._config, "reflexion_max_facts", 8),
+            )
+            return reflexion.refine(source_text, facts)
+        except Exception as exc:
+            logger.debug("Entity reflexion skipped: %s", exc)
+            return facts
+
     def _parse_llm_response(
         self,
         raw: str,

package/src/superlocalmemory/encoding/temporal_validator.py

@@ -96,6 +96,18 @@ class TemporalValidator:
             severity = contradiction["severity"]
             reason = contradiction["description"]
 
+            # Temporal-anchor guard: if the new fact and the old fact carry
+            # distinct explicit event anchors, they describe the same subject at
+            # different times (a valid timeline — "lived in Delhi (2020)" vs
+            # "lives in Mumbai (2024)"), NOT a contradiction. Do not supersede;
+            # both stay recallable so historical queries still find them.
+            if self._is_historical_progression(new_fact, old_fact_id):
+                logger.debug(
+                    "Temporal: %s vs %s have distinct event anchors — historical "
+                    "progression, not superseding", new_fact.fact_id, old_fact_id,
+                )
+                continue
+
             # Step 1: Invalidate the old fact (bi-temporal)
             self.invalidate_fact(
                 fact_id=old_fact_id,
@@ -119,6 +131,54 @@ class TemporalValidator:
         )
         return actions
 
+    @staticmethod
+    def _event_anchor(
+        referenced_date: str | None,
+        observation_date: str | None,
+        interval_start: str | None,
+    ) -> str | None:
+        """The most-specific explicit event time, or None when undated."""
+        return referenced_date or observation_date or interval_start or None
+
+    def _is_historical_progression(
+        self, new_fact: AtomicFact, old_fact_id: str,
+    ) -> bool:
+        """True when new_fact and the old fact have distinct explicit event
+        anchors (same subject, different times) — a valid timeline rather than a
+        contradiction. Conservative: False unless BOTH anchors are present and
+        resolve to different calendar days, so undated facts supersede as before.
+        """
+        new_anchor = self._event_anchor(
+            getattr(new_fact, "referenced_date", None),
+            getattr(new_fact, "observation_date", None),
+            getattr(new_fact, "interval_start", None),
+        )
+        if not new_anchor:
+            return False
+        try:
+            rows = self._db.execute(
+                "SELECT referenced_date, observation_date, interval_start "
+                "FROM atomic_facts WHERE fact_id = ?",
+                (old_fact_id,),
+            )
+        except Exception:
+            return False
+        if not rows:
+            return False
+        d = dict(rows[0])
+        old_anchor = self._event_anchor(
+            d.get("referenced_date"), d.get("observation_date"),
+            d.get("interval_start"),
+        )
+        if not old_anchor:
+            return False
+        from superlocalmemory.retrieval.time_window import parse_timestamp
+        nd = parse_timestamp(new_anchor)
+        od = parse_timestamp(old_anchor)
+        if nd is None or od is None:
+            return False
+        return nd.date() != od.date()
+
     def detect_contradiction(
         self,
         new_fact: AtomicFact,
@@ -206,7 +266,10 @@ class TemporalValidator:
         Both params are REQUIRED in the call site. Do NOT rename to is_valid().
         """
         try:
-            tv = self._db.get_temporal_validity(fact_id)
+            # profile_id or None: an empty-string default must NOT become a
+            # `profile_id = ''` filter (which never matches → an expired fact
+            # would be reported valid). None falls back to the unscoped lookup.
+            tv = self._db.get_temporal_validity(fact_id, profile_id or None)
             if tv is None:
                 return True  # No temporal record = assumed valid
             return (