superlocalmemory 4.1.13 → 4.1.15

package/src/superlocalmemory/learning/entity_compiler.py

@@ -1,356 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Entity Compilation Engine — auto-generates compiled truth per entity.
-
-Builds knowledge summaries using PageRank centrality + Louvain community detection
-(Mode A extractive) or local LLM (Mode B). Per-project, per-profile scoping.
-2000 character hard limit. Read-only layer — never replaces atomic facts.
-
-Runs after consolidation (every 6 hours or on-demand).
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import time
-import uuid
-from datetime import datetime, timezone
-from pathlib import Path
-
-logger = logging.getLogger("superlocalmemory.entity_compiler")
-
-_MAX_COMPILED_TRUTH_CHARS = 2000
-_MAX_TIMELINE_ENTRIES = 100
-
-
-class EntityCompiler:
-    """Compiles knowledge summaries for entities from atomic facts.
-
-    Mode A: Extractive (no LLM) — PageRank + Louvain + top sentences
-    Mode B: Local LLM via Ollama — prompt with top facts
-    """
-
-    def __init__(self, memory_db: str | Path, config=None):
-        self._db_path = str(memory_db)
-        self._config = config
-        self._mode = "a"
-        if config:
-            mode = getattr(config, 'mode', None)
-            if mode:
-                self._mode = getattr(mode, 'value', str(mode)).lower()
-
-    def compile_all(self, profile_id: str) -> dict:
-        """Compile all entities that have new facts across all projects.
-
-        Returns stats: {compiled: N, skipped: N, errors: N}
-
-        Concurrency fix (v3.8.4): each operation uses a scoped connection via
-        memory_read() / memory_write() so the process write lock is never held
-        across Ollama / network calls and no long raw connection is kept open.
-        """
-        from superlocalmemory.storage.memory_write import memory_read
-
-        if self._config and not getattr(self._config, 'entity_compilation_enabled', True):
-            return {"compiled": 0, "skipped": 0, "errors": 0, "reason": "disabled"}
-
-        stats = {"compiled": 0, "skipped": 0, "errors": 0}
-
-        with memory_read(self._db_path) as conn:
-            projects = conn.execute(
-                "SELECT DISTINCT project_name FROM entity_profiles WHERE profile_id = ?",
-                (profile_id,),
-            ).fetchall()
-            project_names = [r[0] for r in projects] if projects else [""]
-
-        for project_name in project_names:
-            result = self._compile_project(profile_id, project_name)
-            stats["compiled"] += result["compiled"]
-            stats["skipped"] += result["skipped"]
-            stats["errors"] += result["errors"]
-
-        if stats["compiled"] > 0:
-            logger.info("Entity compilation: %d compiled, %d skipped, %d errors",
-                        stats["compiled"], stats["skipped"], stats["errors"])
-        return stats
-
-    def compile_entity(self, profile_id: str, project_name: str,
-                       entity_id: str, entity_name: str) -> dict | None:
-        """Compile a single entity. Returns compiled truth or None."""
-        return self._compile_single(profile_id, project_name, entity_id, entity_name)
-
-    def _compile_project(self, profile_id: str, project_name: str) -> dict:
-        """Compile all entities needing update in a project.
-
-        Uses scoped memory_read() for the entity query so no connection is
-        held across entity compilation iterations.
-        """
-        from superlocalmemory.storage.memory_write import memory_read
-
-        stats = {"compiled": 0, "skipped": 0, "errors": 0}
-
-        with memory_read(self._db_path) as conn:
-            entities = conn.execute("""
-                SELECT DISTINCT ce.entity_id, ce.canonical_name, ce.entity_type
-                FROM canonical_entities ce
-                WHERE ce.profile_id = ?
-                AND (
-                    EXISTS (
-                        SELECT 1 FROM atomic_facts af
-                        WHERE af.canonical_entities_json LIKE '%' || ce.entity_id || '%'
-                          AND af.profile_id = ?
-                          AND af.created_at > COALESCE(
-                            (SELECT last_compiled_at FROM entity_profiles
-                             WHERE entity_id = ce.entity_id
-                               AND profile_id = ?
-                               AND project_name = ?),
-                            '1970-01-01')
-                    )
-                    OR NOT EXISTS (
-                        SELECT 1 FROM entity_profiles
-                        WHERE entity_id = ce.entity_id
-                          AND profile_id = ?
-                          AND project_name = ?
-                          AND last_compiled_at IS NOT NULL
-                    )
-                )
-            """, (profile_id, profile_id, profile_id, project_name,
-                  profile_id, project_name)).fetchall()
-            # Detach rows from the read connection before closing it
-            entities = [dict(e) for e in entities]
-
-        for entity in entities:
-            try:
-                result = self._compile_single(
-                    profile_id, project_name,
-                    entity["entity_id"], entity["canonical_name"],
-                    entity_type=entity["entity_type"],
-                )
-                if result:
-                    stats["compiled"] += 1
-                else:
-                    stats["skipped"] += 1
-            except Exception as exc:
-                logger.debug("Entity compilation error for %s: %s",
-                             entity["canonical_name"], exc)
-                stats["errors"] += 1
-
-        return stats
-
-    def _compile_single(self, profile_id: str,
-                         project_name: str, entity_id: str, entity_name: str,
-                         entity_type: str = "unknown") -> dict | None:
-        """Compile one entity. Returns the compiled truth dict or None.
-
-        Concurrency fix (v3.8.4): reads use memory_read(), writes use
-        memory_write().  The Ollama call (Mode B) happens with NO lock held.
-        Lock-ordering invariant preserved: get_write_lock (outermost) →
-        sqlite BEGIN ... COMMIT.
-        """
-        from superlocalmemory.storage.memory_write import memory_read, memory_write
-
-        # ── Phase 1: read facts (no write lock) ──────────────────────────────
-        with memory_read(self._db_path) as conn:
-            facts = conn.execute("""
-                SELECT af.fact_id, af.content, af.confidence, af.created_at,
-                       fi.pagerank_score, fi.community_id
-                FROM atomic_facts af
-                LEFT JOIN fact_importance fi ON af.fact_id = fi.fact_id
-                WHERE af.canonical_entities_json LIKE ? AND af.profile_id = ?
-                ORDER BY fi.pagerank_score DESC NULLS LAST, af.confidence DESC
-                LIMIT 50
-            """, (f"%{entity_id}%", profile_id)).fetchall()
-            facts = [dict(f) for f in facts]
-
-        if not facts:
-            return None
-
-        # There was a second PageRank here, and it made recall worse.
-        #
-        # On finding no score for this entity's facts it built a COMPLETE graph
-        # over them -- every pair joined, weight 0.5 -- and ran PageRank on it.
-        # PageRank of K_n is uniformly 1/n, so every fact received the identical
-        # score and the re-fetch's ORDER BY fell straight through to confidence,
-        # which is what had ordered them in the first place. It bought nothing.
-        #
-        # What it cost: that 1/n landed in the same column the ranker reads as a
-        # whole-graph score. On the author's store, ten facts from one 10-fact
-        # cluster were written 0.1 each, against a real whole-graph maximum of
-        # 0.008744 and median of 0.000214 -- eleven times the largest true score
-        # and roughly 470x the median. The hop boost min(1 + pr*2, 2) gave them
-        # 1.2 where every real memory got 1.0004, and the ten of them together
-        # carried as much PageRank mass as the other 2,988 facts combined (the
-        # table summed to 1.9999 instead of 1). They were also written with no
-        # community, so the community bias could not see them either.
-        #
-        # Whole-graph metrics belong to core/graph_metrics, which owns this
-        # table. Ordering for compilation is confidence, as it always effectively
-        # was.
-
-        # ── Phase 3: generate compiled truth — NO write lock held ────────────
-        # Mode B calls Ollama (up to 30 s) — write lock MUST NOT be held here.
-        if self._mode in ("b", "c") and len(facts) > 3:
-            compiled = self._compile_mode_b(entity_name, facts)
-            if not compiled:
-                compiled = self._compile_mode_a(entity_name, entity_type, facts)
-        else:
-            compiled = self._compile_mode_a(entity_name, entity_type, facts)
-
-        compiled = self._truncate(compiled, _MAX_COMPILED_TRUTH_CHARS)
-
-        now = datetime.now(timezone.utc).isoformat()
-        timeline_entry = {
-            "date": now,
-            "action": "compiled",
-            "facts_used": len(facts),
-            "mode": self._mode,
-        }
-
-        fact_ids = [f["fact_id"] for f in facts]
-        avg_conf = sum(f["confidence"] or 0.5 for f in facts) / max(len(facts), 1)
-
-        # ── Phase 4: short write — hold write lock for INSERT/UPDATE only ────
-        with memory_write(self._db_path) as conn:
-            # Re-read inside the write lock for correct timeline merge and to
-            # decide INSERT vs UPDATE atomically (prevents lost-update race).
-            existing = conn.execute(
-                "SELECT timeline, profile_entry_id FROM entity_profiles "
-                "WHERE entity_id = ? AND profile_id = ? AND project_name = ?",
-                (entity_id, profile_id, project_name),
-            ).fetchone()
-
-            timeline: list = []
-            if existing and existing["timeline"]:
-                try:
-                    timeline = json.loads(existing["timeline"])
-                except (json.JSONDecodeError, TypeError):
-                    timeline = []
-            timeline.append(timeline_entry)
-            if len(timeline) > _MAX_TIMELINE_ENTRIES:
-                timeline = timeline[-_MAX_TIMELINE_ENTRIES:]
-
-            if existing:
-                conn.execute("""
-                    UPDATE entity_profiles SET
-                        compiled_truth = ?, timeline = ?, fact_ids_json = ?,
-                        last_compiled_at = ?, compilation_confidence = ?,
-                        last_updated = ?
-                    WHERE entity_id = ? AND profile_id = ? AND project_name = ?
-                """, (compiled, json.dumps(timeline), json.dumps(fact_ids),
-                      now, round(avg_conf, 3), now,
-                      entity_id, profile_id, project_name))
-            else:
-                entry_id = str(uuid.uuid4())[:16]
-                conn.execute("""
-                    INSERT INTO entity_profiles
-                        (profile_entry_id, entity_id, profile_id, project_name,
-                         knowledge_summary, compiled_truth, timeline, fact_ids_json,
-                         last_compiled_at, compilation_confidence, last_updated)
-                    VALUES (?, ?, ?, ?, '', ?, ?, ?, ?, ?, ?)
-                """, (entry_id, entity_id, profile_id, project_name,
-                      compiled, json.dumps(timeline), json.dumps(fact_ids),
-                      now, round(avg_conf, 3), now))
-
-        return {
-            "entity_name": entity_name,
-            "compiled_truth": compiled,
-            "facts_used": len(facts),
-            "confidence": round(avg_conf, 3),
-        }
-
-    # -- Mode A: Extractive (no LLM) --
-
-    def _compile_mode_a(self, entity_name: str, entity_type: str,
-                         facts: list) -> str:
-        """Extract top sentences by PageRank, grouped by community."""
-        header = f"{entity_name}"
-        if entity_type and entity_type != "unknown":
-            header += f" ({entity_type})"
-        header += "\n"
-
-        # Group facts by community
-        communities: dict[int, list] = {}
-        for f in facts:
-            cid = f["community_id"] or 0
-            communities.setdefault(cid, []).append(f)
-
-        sentences = []
-        seen_content = set()
-        for cid in sorted(communities.keys()):
-            community_facts = communities[cid]
-            # Top 3 facts per community
-            for fact in community_facts[:3]:
-                content = fact["content"]
-                # Extract first sentence
-                first_sent = content.split(". ")[0].strip()
-                if not first_sent.endswith("."):
-                    first_sent += "."
-                # Dedup by exact match
-                normalized = first_sent.lower().strip()
-                if normalized not in seen_content:
-                    seen_content.add(normalized)
-                    sentences.append(first_sent)
-
-        body = " ".join(sentences)
-        return header + body
-
-    # -- Mode B: LLM via Ollama --
-
-    def _compile_mode_b(self, entity_name: str, facts: list) -> str | None:
-        """Summarize via local LLM (Ollama). Returns None on failure."""
-        try:
-            import urllib.request
-            api_base = "http://localhost:11434"
-            if self._config and hasattr(self._config, 'llm'):
-                api_base = getattr(self._config.llm, 'api_base', api_base) or api_base
-            model = "llama3.2"
-            if self._config and hasattr(self._config, 'llm'):
-                model = getattr(self._config.llm, 'model', model) or model
-
-            top_facts = "\n".join(
-                f"- {f['content']}" for f in facts[:20]
-            )
-            prompt = (
-                f"Summarize these facts about {entity_name} into a concise profile. "
-                f"Maximum 2000 characters. Include key relationships, decisions, status. "
-                f"Organize by topic, not chronology. Flag contradictions.\n\n"
-                f"Facts (by importance):\n{top_facts}"
-            )
-
-            payload = json.dumps({
-                "model": model,
-                "prompt": prompt,
-                "stream": False,
-                "options": {"num_predict": 500},
-            }).encode()
-
-            req = urllib.request.Request(
-                f"{api_base}/api/generate",
-                data=payload,
-                headers={"Content-Type": "application/json"},
-            )
-            resp = urllib.request.urlopen(req, timeout=30)
-            result = json.loads(resp.read().decode())
-            text = result.get("response", "").strip()
-            return text if text else None
-        except Exception as exc:
-            logger.debug("Mode B compilation failed, falling back to Mode A: %s", exc)
-            return None
-
-    # -- Helpers --
-
-    @staticmethod
-    def _truncate(text: str, max_chars: int) -> str:
-        """Truncate at sentence boundary within char limit."""
-        if len(text) <= max_chars:
-            return text
-        truncated = text[:max_chars]
-        last_period = truncated.rfind(". ")
-        if last_period > max_chars // 2:
-            return truncated[:last_period + 1]
-        return truncated.rstrip() + "..."

package/src/superlocalmemory/learning/fact_outcome_joins.py

@@ -1,207 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.4.22 — F4.A Stage-8 H-03/H-06 fix
-
-"""Parameterised JSON1-backed join helpers for ``action_outcomes``.
-
-Replaces the fragile ``fact_ids_json LIKE '%"<fid>"%'`` pattern that five
-call sites depended on. Substring matching on serialised JSON leaks
-false positives across overlapping fact_id prefixes — see Stage-8
-skeptic-H06 for the exact failure mode.
-
-This module centralises the correct lookup:
-
-    SELECT outcome_id, ... FROM action_outcomes
-    WHERE profile_id = ?
-      AND EXISTS (
-          SELECT 1 FROM json_each(fact_ids_json) WHERE value = ?
-      )
-
-SQLite ships JSON1 enabled by default since 3.38 (February 2022) and the
-minimum Python supported by SLM is 3.9 — which ships SQLite ≥ 3.31. We
-defensively fall back to a ``LIKE`` probe only when JSON1 is missing at
-runtime, with a one-off warning.
-
-Callers:
-  - ``learning/hnsw_dedup.py`` — ``apply_strong_memory_boost``,
-    ``select_high_reward_fact_ids``, ``run_reward_gated_archive``.
-  - ``learning/forgetting_scheduler.py`` — ``_has_recent_positive_reward``.
-
-Contract refs:
-  - Stage 8 H-03 (architect-H3) + H-06 (skeptic-H06).
-  - LLD-12 §5 — reward-gated archive.
-"""
-
-from __future__ import annotations
-
-import logging
-import sqlite3
-from typing import Iterable
-
-logger = logging.getLogger(__name__)
-
-__all__ = (
-    "iter_outcomes_for_fact",
-    "has_recent_positive_reward",
-    "aggregate_reward_for_fact",
-)
-
-
-# Columns returned — mirror what the legacy LIKE callers read.
-# NB: callers that need extra columns can pass ``columns=``.
-_DEFAULT_COLUMNS = "outcome_id, profile_id, fact_ids_json, reward, settled_at"
-
-
-def _json1_available(conn: sqlite3.Connection) -> bool:
-    """Return True iff SQLite ``json_each`` is usable on ``conn``.
-
-    Result is intentionally not cached across connections — JSON1 is a
-    compile-time flag and runtime-swapping SQLite libraries is a rare
-    edge case but we stay defensive.
-    """
-    try:
-        conn.execute("SELECT value FROM json_each('[\"x\"]') LIMIT 1").fetchall()
-        return True
-    except sqlite3.OperationalError:
-        return False
-
-
-def iter_outcomes_for_fact(
-    conn: sqlite3.Connection,
-    profile_id: str,
-    fact_id: str,
-    *,
-    columns: str = _DEFAULT_COLUMNS,
-    extra_where: str = "",
-    extra_params: tuple = (),
-) -> Iterable[tuple]:
-    """Yield action_outcomes rows whose fact_ids_json contains ``fact_id``.
-
-    Scoped strictly to ``profile_id``; SQL parameters are always bound,
-    never string-interpolated. Returns a materialised list so the caller
-    can close the connection immediately.
-
-    Args:
-        conn: SQLite connection pointing at the database holding the
-            ``action_outcomes`` table (usually ``memory.db``).
-        profile_id: Profile scope.
-        fact_id: Exact fact_id to find.
-        columns: Comma-separated column list projected into the
-            SELECT. Defaults to (outcome_id, profile_id, fact_ids_json,
-            reward, settled_at).
-        extra_where: Optional extra predicate — must start with 'AND'
-            and use '?' placeholders. E.g.
-            ``"AND reward IS NOT NULL AND reward > ?"``.
-        extra_params: Bound parameters for ``extra_where``.
-
-    Returns:
-        List of sqlite3.Row-compatible tuples (or sqlite3.Row objects if
-        the caller set ``conn.row_factory = sqlite3.Row``).
-    """
-    if not profile_id or not fact_id:
-        return []
-
-    if _json1_available(conn):
-        sql = (
-            f"SELECT {columns} FROM action_outcomes "
-            f"WHERE profile_id = ? "
-            f"  AND EXISTS ("
-            f"    SELECT 1 FROM json_each(fact_ids_json) WHERE value = ?"
-            f"  ) "
-            f"{extra_where}"
-        )
-        params = (profile_id, fact_id, *extra_params)
-    else:
-        # Fallback: prefix-LIKE. Accurate ONLY for simple ids.
-        # Logged once per process to flag that JSON1 is missing.
-        _warn_fallback_once()
-        sql = (
-            f"SELECT {columns} FROM action_outcomes "
-            f"WHERE profile_id = ? AND fact_ids_json LIKE ? "
-            f"{extra_where}"
-        )
-        params = (profile_id, f'%"{fact_id}"%', *extra_params)
-
-    cursor = conn.execute(sql, params)
-    return cursor.fetchall()
-
-
-def has_recent_positive_reward(
-    conn: sqlite3.Connection,
-    profile_id: str,
-    fact_id: str,
-    *,
-    min_reward: float = 0.3,
-    window_days: int = 60,
-) -> bool:
-    """True if ``fact_id`` has any outcome with reward > ``min_reward``
-    settled in the last ``window_days`` days.
-    """
-    extra = (
-        "AND reward IS NOT NULL AND reward > ? "
-        f"AND COALESCE(settled_at, '') >= datetime('now', '-{int(window_days)} days') "
-        "LIMIT 1"
-    )
-    rows = iter_outcomes_for_fact(
-        conn, profile_id, fact_id,
-        columns="1",
-        extra_where=extra,
-        extra_params=(float(min_reward),),
-    )
-    return bool(rows)
-
-
-def aggregate_reward_for_fact(
-    conn: sqlite3.Connection,
-    profile_id: str,
-    fact_id: str,
-) -> tuple[int, float]:
-    """Return ``(count, mean_reward)`` for a single fact_id.
-
-    Count is the number of outcomes with reward IS NOT NULL; mean is
-    ``AVG(reward)`` across that same subset. Returns ``(0, 0.0)`` when
-    the fact has no outcomes.
-    """
-    if not profile_id or not fact_id:
-        return 0, 0.0
-
-    if _json1_available(conn):
-        sql = (
-            "SELECT COUNT(*), AVG(reward) FROM action_outcomes "
-            "WHERE profile_id = ? "
-            "  AND reward IS NOT NULL "
-            "  AND EXISTS ("
-            "    SELECT 1 FROM json_each(fact_ids_json) WHERE value = ?"
-            "  )"
-        )
-        row = conn.execute(sql, (profile_id, fact_id)).fetchone()
-    else:
-        _warn_fallback_once()
-        sql = (
-            "SELECT COUNT(*), AVG(reward) FROM action_outcomes "
-            "WHERE profile_id = ? "
-            "  AND reward IS NOT NULL "
-            "  AND fact_ids_json LIKE ?"
-        )
-        row = conn.execute(sql, (profile_id, f'%"{fact_id}"%')).fetchone()
-
-    if row is None:
-        return 0, 0.0
-    count, mean = row
-    return int(count or 0), float(mean or 0.0)
-
-
-_FALLBACK_WARNED = False
-
-
-def _warn_fallback_once() -> None:
-    """Log the JSON1-missing fallback exactly once per process."""
-    global _FALLBACK_WARNED
-    if _FALLBACK_WARNED:
-        return
-    _FALLBACK_WARNED = True
-    logger.warning(
-        "fact_outcome_joins: SQLite JSON1 unavailable — falling back to "
-        "prefix-LIKE. Expect substring false positives on overlapping "
-        "fact_id prefixes. Upgrade SQLite to ≥3.38 for correct matches.",
-    )