superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/encoding/emotional.py

@@ -1,189 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3 — Emotional Tagging (VADER).
-
-Extracts emotional valence and arousal from text.
-Emotionally charged memories are stored more strongly and retrieved more easily
-(amygdala tagging principle from neuroscience).
-
-Ported from V1 — VADER-based, zero-LLM, works in all modes.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import threading
-from dataclasses import dataclass
-
-logger = logging.getLogger(__name__)
-
-# ---------------------------------------------------------------------------
-# Thread-safe, fork-safe VADER singleton
-#
-# Design notes:
-#   • _vader_analyzer = None means "not yet initialized for this process" OR
-#     "initialized but vaderSentiment is unavailable" (both cases return None
-#     from _get_vader, which causes tag_emotion to fall back to the keyword
-#     heuristic — the correct, observable behavior in both cases).
-#
-#   • _vader_pid tracks which OS process last ran initialization. On fork()
-#     (e.g., under pytest-xdist, multiprocessing, or uwsgi) the child inherits
-#     the parent's _vader_analyzer pointer but the native SentimentIntensityAnalyzer
-#     object may reference memory that is no longer safe in the child's address
-#     space. Re-initializing per-PID guarantees each process gets a fresh, safe
-#     object constructed in its own address space.
-#
-#   • _vader_lock (threading.Lock) prevents the double-initialization race in a
-#     multi-threaded caller (e.g., the ingestion thread-pool in store_pipeline.py
-#     where tag_emotion is called from multiple MaterializerWorker threads
-#     concurrently). Without the lock, Thread A and Thread B can both see
-#     _vader_analyzer is None, both enter the import block, and both try to load
-#     vaderSentiment's lexicon simultaneously — causing memory corruption when
-#     the native C-extension in the same process's numpy/BLAS layer is already
-#     running in a third thread (the LanceDB tokio background thread). The lock
-#     serializes VADER initialization so only one thread loads the lexicon.
-#
-#   • After initialization, reads bypass the lock entirely (fast-path at top of
-#     _get_vader). The GIL guarantees atomic reads of _vader_pid and
-#     _vader_analyzer in CPython; writing _vader_pid AFTER _vader_analyzer
-#     (always, inside the lock) ensures a reader that sees a matching _vader_pid
-#     is guaranteed to also see the correctly initialized _vader_analyzer.
-# ---------------------------------------------------------------------------
-
-_vader_analyzer = None          # None = uninitialized OR unavailable
-_vader_pid: int | None = None   # PID in which _vader_analyzer was initialized
-_vader_lock = threading.Lock()  # Serializes initialization only; not scoring
-
-
-def _get_vader():
-    """Lazy-load VADER — thread-safe via lock, fork-safe via PID tracking.
-
-    Returns a SentimentIntensityAnalyzer instance, or None if vaderSentiment
-    is not installed.  Callers must treat None as "fallback to keyword heuristic."
-    """
-    global _vader_analyzer, _vader_pid
-    current_pid = os.getpid()
-
-    # Fast-path: already initialized in this process — no lock needed.
-    # We read _vader_pid first; if it matches current_pid, _vader_analyzer was
-    # written before _vader_pid was set (see ordering in slow-path below),
-    # so the value we read is fully initialized.
-    if _vader_pid == current_pid:
-        return _vader_analyzer
-
-    # Slow-path: either first call in this process, or a forked child.
-    with _vader_lock:
-        # Double-checked locking: another thread may have raced us here.
-        if _vader_pid == current_pid:
-            return _vader_analyzer
-
-        # Initialize (or re-initialize) for this process.
-        try:
-            import warnings
-            with warnings.catch_warnings():
-                warnings.filterwarnings(
-                    "ignore", category=DeprecationWarning, module="vaderSentiment",
-                )
-                from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
-                _vader_analyzer = SentimentIntensityAnalyzer()
-        except ImportError:
-            logger.warning("vaderSentiment not installed — emotional tagging disabled")
-            _vader_analyzer = None
-
-        # Write _vader_pid LAST so that the fast-path can safely use it as a
-        # "initialization complete" signal.
-        _vader_pid = current_pid
-
-    return _vader_analyzer
-
-
-@dataclass(frozen=True)
-class EmotionalTag:
-    """Emotional metadata for a memory or fact."""
-
-    valence: float   # -1.0 (negative) to +1.0 (positive)
-    arousal: float   # 0.0 (calm) to 1.0 (intense)
-
-
-def tag_emotion(text: str) -> EmotionalTag:
-    """Extract emotional valence and arousal from text.
-
-    Valence: VADER compound score [-1, +1].
-    Arousal: absolute compound + max(pos, neg) — higher = more emotional intensity.
-    Falls back to keyword heuristic when VADER is unavailable.
-    """
-    analyzer = _get_vader()
-    if analyzer is None:
-        return _keyword_fallback(text)
-
-    scores = analyzer.polarity_scores(text)
-    compound = scores["compound"]     # -1 to +1
-    pos = scores["pos"]               # 0 to 1
-    neg = scores["neg"]               # 0 to 1
-
-    valence = compound
-    # Arousal = emotional intensity regardless of direction
-    arousal = min(1.0, abs(compound) * 0.6 + max(pos, neg) * 0.4)
-
-    return EmotionalTag(valence=round(valence, 4), arousal=round(arousal, 4))
-
-
-_POSITIVE_WORDS: frozenset[str] = frozenset({
-    "love", "amazing", "wonderful", "great", "happy", "fantastic",
-    "excellent", "beautiful", "awesome", "brilliant", "incredible",
-    "joy", "thrilled", "grateful", "delighted", "superb",
-})
-
-_NEGATIVE_WORDS: frozenset[str] = frozenset({
-    "hate", "terrible", "horrible", "awful", "bad", "worst",
-    "angry", "frustrated", "disappointed", "sad", "miserable",
-    "disgusting", "dreadful", "pathetic", "furious", "outraged",
-})
-
-
-def _keyword_fallback(text: str) -> EmotionalTag:
-    """Lightweight sentiment heuristic when VADER is unavailable.
-
-    Counts positive/negative keywords and derives approximate valence/arousal.
-    """
-    if not text.strip():
-        return EmotionalTag(valence=0.0, arousal=0.0)
-
-    words = set(text.lower().split())
-    pos_count = len(words & _POSITIVE_WORDS)
-    neg_count = len(words & _NEGATIVE_WORDS)
-    total = pos_count + neg_count
-
-    if total == 0:
-        return EmotionalTag(valence=0.0, arousal=0.0)
-
-    # Valence: positive - negative, normalised to [-1, 1]
-    raw_valence = (pos_count - neg_count) / total
-    valence = max(-1.0, min(1.0, raw_valence))
-
-    # Arousal: how many emotional words relative to total word count
-    word_count = max(len(words), 1)
-    arousal = min(1.0, total / word_count * 2.0)
-
-    return EmotionalTag(valence=round(valence, 4), arousal=round(arousal, 4))
-
-
-def is_emotionally_significant(tag: EmotionalTag, threshold: float = 0.3) -> bool:
-    """Check if the emotional signal is strong enough to boost importance."""
-    return tag.arousal >= threshold
-
-
-def emotional_importance_boost(tag: EmotionalTag) -> float:
-    """Compute importance boost from emotional signal.
-
-    Returns 0.0-0.3 boost. High arousal memories get stored more strongly
-    (amygdala-inspired encoding enhancement).
-    """
-    if tag.arousal <= 0.2:
-        return 0.0
-    return min(0.3, tag.arousal * 0.3)

package/src/superlocalmemory/encoding/entity_reflexion.py

@@ -1,200 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Reflexion self-review over LLM-extracted entities (Wave Q1).
-
-After primary LLM extraction (Mode B/C only), one bounded self-review pass
-audits the extracted entities against the source text and:
-
-  - drops hallucinated entities (not supported by the source), and
-  - adds clearly-missed entities (whose exact text appears in the source).
-
-This closes the two distinct extraction failure modes the market leaders
-separate: coverage (missed) and grounding (hallucinated). Coverage is the
-"gleaning" idea (re-present what was extracted, ask for misses); grounding is
-the "reflexion" idea (verify each extracted entity is real).
-
-Design guarantees:
-  - Mode A never reaches here — the caller gates on Mode B/C + LLM available.
-  - Fail-open: any error, unavailable LLM, or unparseable output returns the
-    input facts unchanged. This module NEVER raises.
-  - Immutable: corrected facts are new objects; inputs are not mutated.
-  - Bounded: one LLM call, capped facts reviewed, capped entities per fact.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import re
-from dataclasses import replace
-from typing import Any
-
-from superlocalmemory.storage.models import AtomicFact
-
-logger = logging.getLogger(__name__)
-
-_MAX_ENTITIES_PER_FACT = 12
-_MIN_ENTITY_LEN = 2
-
-_SYSTEM = (
-    "You are a precise information-extraction auditor. You verify that "
-    "entities extracted from text are actually supported by that text, and "
-    "you catch clearly-named entities that were missed. You never invent "
-    "entities. You reply with JSON only."
-)
-
-
-class EntityReflexion:
-    """Bounded, fail-open reflexion pass over extracted entities."""
-
-    def __init__(
-        self,
-        llm: Any,
-        max_facts: int = 8,
-        max_tokens: int = 512,
-    ) -> None:
-        self._llm = llm
-        self._max_facts = max(1, int(max_facts))
-        self._max_tokens = max(64, int(max_tokens))
-
-    # ------------------------------------------------------------------
-    # Public API
-    # ------------------------------------------------------------------
-
-    def refine(
-        self, source_text: str, facts: list[AtomicFact],
-    ) -> list[AtomicFact]:
-        """Return facts with entity lists corrected, or unchanged on failure."""
-        if not self._llm or not facts or not source_text or not source_text.strip():
-            return facts
-        try:
-            is_avail = getattr(self._llm, "is_available", None)
-            if callable(is_avail) and not is_avail():
-                return facts
-        except Exception:
-            return facts
-
-        subject = facts[: self._max_facts]
-        try:
-            raw = self._invoke(source_text, subject)
-        except Exception as exc:
-            logger.debug("EntityReflexion LLM call failed: %s", exc)
-            return facts
-
-        corrections = self._parse(raw)
-        if not corrections:
-            return facts
-        try:
-            return self._apply(facts, corrections, source_text)
-        except Exception as exc:
-            logger.debug("EntityReflexion apply failed: %s", exc)
-            return facts
-
-    # ------------------------------------------------------------------
-    # Internal
-    # ------------------------------------------------------------------
-
-    def _invoke(self, source_text: str, subject: list[AtomicFact]) -> str:
-        lines = []
-        for i, f in enumerate(subject):
-            lines.append(f'{i}: "{f.content}" entities={list(f.entities)}')
-        facts_block = "\n".join(lines)
-        prompt = (
-            "Audit the extracted entities for each fact against the SOURCE "
-            "text below.\n"
-            "For each fact, report:\n"
-            "- drop: entities in the fact's list that are NOT supported by the "
-            "source (hallucinated).\n"
-            "- add: named entities clearly present in the source but missing "
-            "from the fact's list. Only add entities whose exact text appears "
-            "in the source.\n\n"
-            f"--- SOURCE ---\n{source_text}\n--- END ---\n\n"
-            f"--- FACTS ---\n{facts_block}\n--- END ---\n\n"
-            'Respond with ONLY a JSON array: '
-            '[{"index": <int>, "drop": [..], "add": [..]}]. '
-            "Use [] for an empty list. Omit facts that need no change."
-        )
-        out = self._llm.generate(
-            prompt=prompt,
-            system=_SYSTEM,
-            temperature=0.0,
-            max_tokens=self._max_tokens,
-        )
-        return out if isinstance(out, str) else str(out or "")
-
-    @staticmethod
-    def _parse(raw: str) -> list[dict]:
-        if not raw or not raw.strip():
-            return []
-        match = re.search(r"\[.*\]", raw, re.DOTALL)
-        if not match:
-            return []
-        try:
-            data = json.loads(match.group())
-        except (json.JSONDecodeError, ValueError):
-            return []
-        if not isinstance(data, list):
-            return []
-        return [d for d in data if isinstance(d, dict)]
-
-    def _apply(
-        self,
-        facts: list[AtomicFact],
-        corrections: list[dict],
-        source_text: str,
-    ) -> list[AtomicFact]:
-        by_index: dict[int, dict] = {}
-        for c in corrections:
-            idx = c.get("index")
-            if isinstance(idx, int) and 0 <= idx < self._max_facts:
-                by_index[idx] = c
-
-        src_low = source_text.lower()
-        out: list[AtomicFact] = []
-        for i, fact in enumerate(facts):
-            c = by_index.get(i)
-            if c is None:
-                out.append(fact)
-                continue
-
-            ents = list(fact.entities)
-            drop = {
-                str(d).strip().lower()
-                for d in _as_list(c.get("drop"))
-                if str(d).strip()
-            }
-            if drop:
-                ents = [e for e in ents if e.strip().lower() not in drop]
-
-            seen = {e.strip().lower() for e in ents}
-            for a in _as_list(c.get("add")):
-                cand = str(a).strip()
-                low = cand.lower()
-                if len(cand) < _MIN_ENTITY_LEN or low in seen:
-                    continue
-                # Grounding guard: only add entities literally present in source.
-                if low not in src_low:
-                    continue
-                ents.append(cand)
-                seen.add(low)
-                if len(ents) >= _MAX_ENTITIES_PER_FACT:
-                    break
-
-            if ents != list(fact.entities):
-                out.append(replace(fact, entities=ents))
-            else:
-                out.append(fact)
-        return out
-
-
-def _as_list(value: Any) -> list:
-    if isinstance(value, list):
-        return value
-    if value is None:
-        return []
-    return [value]