superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/core/memory_health.py

@@ -1,266 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""Tell the owner, in their own words, whether their memory works.
-
-Until now the only way to learn that 43.7% of a store could not be found by
-asking a question was to write the SQL yourself. One machine sat in exactly
-that state for months while every status line it showed said the system was
-healthy, because nothing measured reachability and nothing reported it.
-
-So this module answers four questions a non-engineer can act on:
-
-  * How many memories do I have?
-  * How many can actually be found by asking a question?
-  * How many were withheld because a model wrote them, not me?
-  * Is anything still being repaired?
-
-Read-only, and every query is bounded. Fail-soft by construction: a missing
-table or column yields ``None`` for that line rather than an exception, because
-a health report that crashes on an old store is worse than one that says "not
-known yet".
-
-Consumed by ``slm doctor``, ``GET /api/v3/memory-health``, and the dashboard.
-One implementation so the three cannot disagree with each other.
-"""
-
-from __future__ import annotations
-
-import logging
-import re
-import sqlite3
-from dataclasses import dataclass, field
-from pathlib import Path
-
-logger = logging.getLogger(__name__)
-
-__all__ = ["MemoryHealth", "measure", "describe"]
-
-
-@dataclass(frozen=True)
-class MemoryHealth:
-    """A store's answer-ability, counted rather than assumed."""
-
-    #: Memories that recall is allowed to return.
-    live_facts: int = 0
-    #: Of those, how many have a vector projection, i.e. can be found by
-    #: meaning rather than only by matching words.
-    findable_by_meaning: int = 0
-    #: Memories with no vector at all. These are reachable by keyword only.
-    missing_vector: int = 0
-    #: Machine-written summaries withheld from recall and kept for display.
-    withheld_summaries: int = 0
-    #: Summaries preserved in the display table.
-    display_summaries: int = 0
-    #: Memories hidden by the retention system, excluding the withheld ones.
-    hidden_by_forgetting: int = 0
-    #: Rows whose retention zone contradicts their retention score, i.e. hidden
-    #: while the maths says to keep them. Should be zero after repair.
-    inconsistently_hidden: int = 0
-    #: Present only when a table or column was absent.
-    unavailable: tuple[str, ...] = field(default_factory=tuple)
-
-    @property
-    def reachability(self) -> float:
-        """Share of live memories findable by meaning, 0.0-1.0."""
-        if self.live_facts <= 0:
-            return 1.0
-        return self.findable_by_meaning / self.live_facts
-
-    @property
-    def healthy(self) -> bool:
-        """Whether anything here warrants telling the owner about."""
-        return (
-            self.reachability >= 0.99
-            and self.missing_vector == 0
-            and self.inconsistently_hidden == 0
-        )
-
-
-def measure(db_path: str | Path) -> MemoryHealth:
-    """Count the store's answer-ability. Read-only; never raises."""
-    unavailable: list[str] = []
-    try:
-        conn = sqlite3.connect(f"file:{Path(db_path)}?mode=ro", uri=True)
-    except sqlite3.Error as exc:
-        logger.debug("memory health: cannot open %s: %s", db_path, exc)
-        return MemoryHealth(unavailable=("database",))
-
-    try:
-        if not _table_exists(conn, "atomic_facts"):
-            return MemoryHealth(unavailable=("atomic_facts",))
-
-        # Quarantine came in 4.0.10. On an older store every fact is "live",
-        # which is the honest reading of a store that has no withheld rows.
-        has_q = _has_column(conn, "atomic_facts", "quarantined")
-        if not has_q:
-            unavailable.append("quarantined")
-        live_clause = "COALESCE(quarantined, 0) = 0" if has_q else "1=1"
-
-        live = _count(conn, f"SELECT COUNT(*) FROM atomic_facts WHERE {live_clause}")
-        withheld = (
-            _count(conn, "SELECT COUNT(*) FROM atomic_facts WHERE quarantined = 1")
-            if has_q else 0
-        )
-        missing_vec = _count(
-            conn,
-            f"SELECT COUNT(*) FROM atomic_facts "
-            f"WHERE embedding IS NULL AND {live_clause}",
-        )
-
-        if _table_exists(conn, "embedding_metadata"):
-            findable = _count(
-                conn,
-                "SELECT COUNT(*) FROM embedding_metadata em "
-                "JOIN atomic_facts af ON af.fact_id = em.fact_id "
-                f"WHERE {_prefixed(live_clause, 'af')}",
-            )
-        else:
-            unavailable.append("embedding_metadata")
-            findable = 0
-
-        display = (
-            _count(conn, "SELECT COUNT(*) FROM consolidated_summaries")
-            if _table_exists(conn, "consolidated_summaries") else 0
-        )
-        if not _table_exists(conn, "consolidated_summaries"):
-            unavailable.append("consolidated_summaries")
-
-        hidden = inconsistent = 0
-        if _table_exists(conn, "fact_retention"):
-            hidden = _count(
-                conn,
-                "SELECT COUNT(*) FROM fact_retention r "
-                "JOIN atomic_facts af ON af.fact_id = r.fact_id "
-                "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
-                f"  AND {_prefixed(live_clause, 'af')}",
-            )
-            # The contradiction M043 repairs: hidden, yet scored to keep.
-            inconsistent = _count(
-                conn,
-                "SELECT COUNT(*) FROM fact_retention r "
-                "JOIN atomic_facts af ON af.fact_id = r.fact_id "
-                "WHERE r.lifecycle_zone IN ('archive', 'forgotten') "
-                "  AND r.retention_score > 0.8 "
-                f"  AND {_prefixed(live_clause, 'af')}",
-            )
-        else:
-            unavailable.append("fact_retention")
-
-        return MemoryHealth(
-            live_facts=live,
-            findable_by_meaning=findable,
-            missing_vector=missing_vec,
-            withheld_summaries=withheld,
-            display_summaries=display,
-            hidden_by_forgetting=hidden,
-            inconsistently_hidden=inconsistent,
-            unavailable=tuple(unavailable),
-        )
-    except sqlite3.Error as exc:
-        logger.debug("memory health measurement failed: %s", exc)
-        return MemoryHealth(unavailable=(*unavailable, "query_failed"))
-    finally:
-        conn.close()
-
-
-def describe(health: MemoryHealth) -> list[str]:
-    """Plain-language lines for a reader who does not write SQL.
-
-    No percentages without the counts behind them, and no jargon: "findable by
-    asking a question" rather than "vector coverage", because the person who
-    needs this line is the one who would not know what a vector is.
-    """
-    lines: list[str] = []
-    if "atomic_facts" in health.unavailable or "database" in health.unavailable:
-        return ["Memory store not readable yet."]
-
-    lines.append(f"You have {health.live_facts:,} memories.")
-
-    if "embedding_metadata" in health.unavailable:
-        lines.append(
-            "Whether they can be found by asking a question is not known yet — "
-            "the search index has not been built."
-        )
-    elif health.live_facts:
-        pct = 100.0 * health.reachability
-        if health.findable_by_meaning >= health.live_facts:
-            # "All" only when the counts actually agree. The threshold used to
-            # be reachability >= 0.99, which printed "All of them can be found
-            # by asking a question (5,199 indexed)" on a store of 5,205 — a
-            # claim of all, contradicted by the number beside it. This module
-            # exists to be believed; it cannot round in its own favour.
-            lines.append(
-                f"All {health.live_facts:,} of them can be found by asking a "
-                f"question."
-            )
-        elif health.reachability >= 0.99:
-            gap = health.live_facts - health.findable_by_meaning
-            lines.append(
-                f"{health.findable_by_meaning:,} of them can be found by asking "
-                f"a question. The other {gap:,} can only be found by matching "
-                f"words. That is a small enough share to be normal — a memory "
-                f"written moments ago, or one the model could not read."
-            )
-        else:
-            gap = health.live_facts - health.findable_by_meaning
-            lines.append(
-                f"{health.findable_by_meaning:,} of them ({pct:.0f}%) can be "
-                f"found by asking a question. The other {gap:,} can only be "
-                f"found by matching words, so a question phrased differently "
-                f"will miss them. This repairs itself as the service runs; if "
-                f"it does not, the embedding model is unavailable."
-            )
-
-    if health.withheld_summaries:
-        lines.append(
-            f"{health.withheld_summaries:,} machine-written summaries are kept "
-            f"out of your answers and shown on the dashboard instead. They were "
-            f"written by the summarizer, not by you, and they used to be "
-            f"returned as if they were your own notes."
-        )
-
-    if health.inconsistently_hidden:
-        lines.append(
-            f"{health.inconsistently_hidden:,} memories are hidden even though "
-            f"they are marked worth keeping. This is a fault and it is repaired "
-            f"automatically the next time the service starts."
-        )
-
-    if health.hidden_by_forgetting:
-        lines.append(
-            f"{health.hidden_by_forgetting:,} older memories are set aside by "
-            f"the forgetting curve. They are not deleted and a deep search "
-            f"still reaches them."
-        )
-
-    return lines
-
-
-def _table_exists(conn: sqlite3.Connection, table: str) -> bool:
-    return conn.execute(
-        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,),
-    ).fetchone() is not None
-
-
-def _has_column(conn: sqlite3.Connection, table: str, column: str) -> bool:
-    return any(
-        row[1] == column for row in conn.execute(f"PRAGMA table_info({table})")
-    )
-
-
-def _prefixed(clause: str, prefix: str) -> str:
-    """Qualify a bare column reference for use in a joined query.
-
-    Word-bounded, so a future column named ``quarantined_at`` is not silently
-    rewritten to ``af.quarantined_at`` by a substring match. No such column
-    exists today; the point is that the failure would be a wrong count rather
-    than an error, and a wrong count in a health report is the one thing this
-    module must not produce.
-    """
-    return re.sub(r"\bquarantined\b", f"{prefix}.quarantined", clause)
-
-
-def _count(conn: sqlite3.Connection, sql: str) -> int:
-    row = conn.execute(sql).fetchone()
-    return int(row[0]) if row else 0

package/src/superlocalmemory/core/mode_capability.py

@@ -1,111 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""What the running mode can do with a model, in words a user can act on.
-
-WHY THIS EXISTS
-
-Mode A runs with no language model at all. That is the point of it: nothing on
-the store or recall path calls out to anything, and a summary is assembled
-directly from the user's own notes rather than written.
-
-The surfaces already reported *what* they did -- a summary came back labelled
-"assembled directly from your own notes" -- but never *why*, and never what to do
-about it. Someone looking at a plainer summary than they expected had no way to
-learn that a written one needs a model and which modes have one. They would
-reasonably conclude the feature was broken.
-
-So this returns the mode, whether a model is available, and one sentence naming
-the next step. Every model-backed surface returns the same block, so the
-explanation is identical wherever it appears rather than reworded per pane.
-
-WHAT IT IS NOT
-
-It is not a capability gate. Nothing here decides whether a call is made; the
-mode's configuration does that. This only describes the decision to the person
-looking at the result.
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-logger = logging.getLogger(__name__)
-
-#: Named so the message can say which modes to consider without hard-coding the
-#: sentence at each call site.
-_MODE_WITH_LOCAL_MODEL = "B"
-_MODE_WITH_CLOUD_MODEL = "C"
-
-_NO_MODEL_MESSAGE = (
-    "This mode runs entirely on your machine with no language model, so "
-    "summaries are assembled from your own notes rather than written. For "
-    f"written summaries and the other model-backed features, switch to Mode "
-    f"{_MODE_WITH_LOCAL_MODEL} (a local model) or Mode {_MODE_WITH_CLOUD_MODEL} "
-    "(your own cloud model) in Settings and connect one."
-)
-
-_MODEL_CONFIGURED_BUT_ABSENT = (
-    "This mode uses a language model, but none is reachable right now, so "
-    "results fall back to being assembled from your own notes. Check the model "
-    "settings, and that the local model server is running if you are using one."
-)
-
-_CLOUD_MODEL_HAS_NO_KEY = (
-    "This mode uses a hosted language model and no key has been set for it, so "
-    "summaries are assembled from your own notes rather than written. Run "
-    "`slm provider set` and supply your key, or switch to Mode "
-    f"{_MODE_WITH_LOCAL_MODEL} to use a model on this machine."
-)
-
-#: Providers that run on the machine and need no credential. Anything else is
-#: a hosted service, and naming one without a key is not a configured model --
-#: which is what this used to report, so the surfaces said everything was fine
-#: while every model-backed feature was about to fall back.
-_LOCAL_PROVIDERS = frozenset({
-    "ollama", "lmstudio", "llamacpp", "llama.cpp", "local", "vllm", "none", "",
-})
-
-
-def llm_capability(config: Any, *, llm_reachable: bool | None = None) -> dict:
-    """Describe this mode's model support for a user-facing surface.
-
-    ``llm_reachable`` lets a caller that already knows the answer pass it in --
-    a summary route has just tried and knows whether it worked, and asking again
-    would mean a second connection attempt to say something it already knows.
-    Left as None, availability is taken from configuration alone.
-    """
-    mode = ""
-    provider = ""
-    api_key = ""
-    try:
-        mode = str(getattr(getattr(config, "mode", None), "value", "") or "").upper()
-        llm = getattr(config, "llm", None)
-        provider = str(getattr(llm, "provider", "") or "")
-        api_key = str(getattr(llm, "api_key", "") or "").strip()
-    except Exception as exc:  # noqa: BLE001 -- a description must not raise
-        logger.debug("mode capability: cannot read config: %s", exc)
-
-    mode_has_model = mode in (_MODE_WITH_LOCAL_MODEL, _MODE_WITH_CLOUD_MODEL)
-    needs_a_key = provider.lower() not in _LOCAL_PROVIDERS
-    missing_key = mode_has_model and bool(provider) and needs_a_key and not api_key
-    configured = bool(provider) and mode_has_model and not missing_key
-    available = configured if llm_reachable is None else bool(llm_reachable)
-
-    if not mode_has_model:
-        message = _NO_MODEL_MESSAGE
-    elif missing_key and llm_reachable is not True:
-        message = _CLOUD_MODEL_HAS_NO_KEY
-    elif not available:
-        message = _MODEL_CONFIGURED_BUT_ABSENT
-    else:
-        message = ""
-
-    return {
-        "mode": mode or "?",
-        "llm_available": bool(available),
-        "llm_provider": provider,
-        "message": message,
-    }

package/src/superlocalmemory/core/modes.py

@@ -1,168 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V4 mode capability descriptors.
-
-Three operating modes with clear capability boundaries.
-Mode A: deterministic local extraction and local inference.
-Mode B: local LLM enrichment and local inference.
-Mode C: configured provider-assisted inference.
-
-An operating mode does not determine EU AI Act compliance.  That assessment
-depends on the deployment, intended use, operator role, and applicable duties.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-
-from superlocalmemory.storage.models import Mode
-
-
-@dataclass(frozen=True)
-class ModeCapabilities:
-    """What each mode can and cannot do."""
-
-    mode: Mode
-
-    # Encoding capabilities
-    llm_fact_extraction: bool      # Can use LLM for fact extraction?
-    llm_entity_resolution: bool    # Can use LLM for entity disambiguation?
-    llm_type_classification: bool  # Can use LLM for fact type routing?
-    llm_importance_scoring: bool   # Can use LLM for importance assessment?
-
-    # Retrieval capabilities
-    agentic_retrieval: bool        # Can do multi-round LLM-guided retrieval?
-    llm_answer_generation: bool    # Can LLM generate answers from context?
-    cloud_reranker: bool           # Can use Cohere / cloud reranker?
-
-    # Embedding capabilities
-    cloud_embeddings: bool         # Can use cloud embedding API?
-    embedding_dimension: int       # Expected embedding dimension
-
-    # Compliance
-    eu_ai_act_compliant: bool | None  # None: requires deployment assessment
-    data_stays_local: bool         # Does ALL data stay on device?
-
-    # Description
-    description: str = ""
-
-    @property
-    def data_locality_label(self) -> str:
-        """UI/API locality label derived from the mode record (single source).
-
-        Never invent a parallel mode→label map in the dashboard — consume this.
-        """
-        return "local-only" if self.data_stays_local else "provider-assisted"
-
-
-# ---------------------------------------------------------------------------
-# Mode Definitions
-# ---------------------------------------------------------------------------
-
-MODE_A = ModeCapabilities(
-    mode=Mode.A,
-    llm_fact_extraction=False,
-    llm_entity_resolution=False,
-    llm_type_classification=False,
-    llm_importance_scoring=False,
-    agentic_retrieval=False,
-    llm_answer_generation=False,
-    cloud_reranker=False,
-    cloud_embeddings=False,
-    embedding_dimension=768,
-    eu_ai_act_compliant=None,
-    data_stays_local=True,
-    description=(
-        "Local Guardian — Zero LLM, zero cloud. "
-        "Uses nomic-embed-text-v1.5 encoder (768d, 8K context) for embeddings. "
-        "Deterministic rules for extraction and a local PyTorch cross-encoder. "
-        "EU AI Act classification requires deployment assessment."
-    ),
-)
-
-MODE_B = ModeCapabilities(
-    mode=Mode.B,
-    llm_fact_extraction=True,
-    llm_entity_resolution=True,
-    llm_type_classification=True,
-    llm_importance_scoring=True,
-    agentic_retrieval=False,
-    llm_answer_generation=True,
-    cloud_reranker=False,
-    cloud_embeddings=False,
-    embedding_dimension=768,
-    eu_ai_act_compliant=None,
-    data_stays_local=True,
-    description=(
-        "Smart Local — Local Ollama LLM (Phi-3, Llama 3.2). "
-        "LLM-quality extraction and classification, fully local. "
-        "Local PyTorch cross-encoder reranking. No configured cloud inference. "
-        "EU AI Act classification requires deployment assessment."
-    ),
-)
-
-MODE_C = ModeCapabilities(
-    mode=Mode.C,
-    llm_fact_extraction=True,
-    llm_entity_resolution=True,
-    llm_type_classification=True,
-    llm_importance_scoring=True,
-    agentic_retrieval=True,
-    llm_answer_generation=True,
-    cloud_reranker=True,
-    cloud_embeddings=True,
-    embedding_dimension=3072,
-    eu_ai_act_compliant=None,
-    data_stays_local=False,
-    description=(
-        "FULL POWER — UNRESTRICTED. Best embeddings (text-embedding-3-large, 3072-dim). "
-        "Best configured cloud LLMs (e.g. GPT-5, Claude Opus 4). Agentic multi-round retrieval. "
-        "Cohere reranker option. Cloud processing requires deployment-specific "
-        "privacy, contractual, and EU AI Act assessment."
-    ),
-)
-
-
-def get_capabilities(mode: Mode) -> ModeCapabilities:
-    """Get capability matrix for a mode."""
-    _map = {Mode.A: MODE_A, Mode.B: MODE_B, Mode.C: MODE_C}
-    return _map[mode]
-
-
-def dashboard_mode_fields(mode: Mode | str) -> dict[str, object]:
-    """Fields the dashboard API must surface from the mode record.
-
-    Keeps UI locality claims bound to :class:`ModeCapabilities` so Mode C
-    cannot be labeled local-only by a divergent hardcode.
-    """
-    if isinstance(mode, str):
-        mode = Mode(mode.strip().lower())
-    caps = get_capabilities(mode)
-    return {
-        "data_locality_label": caps.data_locality_label,
-        "data_stays_local": caps.data_stays_local,
-    }
-
-
-def validate_mode_config(mode: Mode, *, has_ollama: bool = False, has_cloud_llm: bool = False) -> list[str]:
-    """Validate that required services are available for the chosen mode.
-
-    Returns list of warnings/errors. Empty list = all good.
-    """
-    issues: list[str] = []
-    caps = get_capabilities(mode)
-
-    if caps.llm_fact_extraction and mode == Mode.B and not has_ollama:
-        issues.append("Mode B requires Ollama but it is not available. Falling back to Mode A extraction.")
-
-    if caps.cloud_embeddings and not has_cloud_llm:
-        issues.append("Mode C cloud embeddings configured but no API endpoint provided.")
-
-    if caps.agentic_retrieval and not has_cloud_llm:
-        issues.append("Mode C agentic retrieval requires cloud LLM but none configured.")
-
-    return issues