superlocalmemory 4.0.10 → 4.1.2

package/src/superlocalmemory/core/scale_autopromote.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Move an existing store onto the graph and vector backends, by itself.
+
+Cozo (graph) and LanceDB (vectors) have shipped as required dependencies since
+3.7, and until now they sat unused: the projections were only built when
+somebody ran `slm db scale prepare` and then `verify` and then `promote` by
+hand. Almost nobody did, so almost every store kept answering graph and
+similarity questions out of SQLite.
+
+This runs that same sequence on the first start after an upgrade, so an existing
+user gets the backends without doing anything.
+
+WHAT IT WILL NOT DO
+
+**It will not leave a store worse than it found it.** The lifecycle it drives
+already stages, verifies against the canonical counts, and keeps a rollback —
+this only decides when to call it. If verification does not match, nothing is
+promoted and the store keeps answering from SQLite.
+
+**It will not stop a store from working.** If the libraries are absent or fail
+to import, or the projection cannot be built, the daemon serves from SQLite and
+says so. Refusing to start would be a worse outcome than the one being fixed:
+a user whose native extension does not match their interpreter would have no
+product at all, and that is not a hypothetical — it was the state of the
+author's own machine while this was written.
+
+**It will not run twice.** Once the state is `promoted` there is nothing to do,
+and a store mid-repair is left for the repair path rather than restarted.
+
+WHAT IT COSTS, MEASURED
+
+On a real 604 MB store — 5,283 memories, 137,104 edges, 395,107 links, 5,278
+vectors — prepare took 10 s, verify 5 s, promote 5 s, and the two projections
+occupy 16 MB each. It happens once.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["AutoPromotionResult", "auto_promote_scale_backends"]
+
+#: The two the projections need. Absence is a reason to stay on SQLite, never a
+#: reason to fail.
+REQUIRED_LIBRARIES = ("pycozo", "lancedb")
+
+
+@dataclass(frozen=True)
+class AutoPromotionResult:
+    """What happened, in terms a status endpoint can show a person."""
+
+    attempted: bool
+    promoted: bool
+    reason: str
+    stage_id: str = ""
+    restart_required: bool = False
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "attempted": self.attempted,
+            "promoted": self.promoted,
+            "reason": self.reason,
+            "stage_id": self.stage_id,
+            "restart_required": self.restart_required,
+        }
+
+
+def _missing_libraries() -> list[str]:
+    import importlib
+
+    missing: list[str] = []
+    for name in REQUIRED_LIBRARIES:
+        try:
+            importlib.import_module(name)
+        except Exception:  # noqa: BLE001 - any import failure means unusable
+            missing.append(name)
+    return missing
+
+
+def _resumable_stage(status: dict[str, Any]) -> str:
+    """The newest stage already built and not yet promoted, if there is one.
+
+    A stage that is `prepared` or `verified` represents work already done
+    against the same store. Building another one repeats it and leaves the
+    first behind.
+    """
+    usable = [
+        stage for stage in (status.get("stages") or [])
+        if str(stage.get("state")) in {"prepared", "verified"}
+        and stage.get("stage_id")
+    ]
+    if not usable:
+        return ""
+    usable.sort(key=lambda stage: str(stage.get("created_at", "")))
+    return str(usable[-1]["stage_id"])
+
+
+def auto_promote_scale_backends(config: Any) -> AutoPromotionResult:
+    """Build and promote the projections if this store has not got them yet.
+
+    Returns what happened rather than raising: every outcome here is a state the
+    daemon carries on from, and the caller shows it.
+    """
+    state = str(getattr(config, "scale_engine_state", "") or "local_core").lower()
+    if state == "promoted":
+        return AutoPromotionResult(False, True, "already promoted")
+
+    # A switch that exists and is honoured elsewhere is a switch this must
+    # honour too. The background path has read it since 3.7; starting to move a
+    # user's store on a setting that says not to would be worse than never
+    # having automated it.
+    if not bool(getattr(config, "scale_auto_promote_enabled", True)):
+        return AutoPromotionResult(
+            False, False, "automatic promotion is switched off in the configuration",
+        )
+
+    missing = _missing_libraries()
+    if missing:
+        logger.warning(
+            "graph and vector backends unavailable (%s will not import); "
+            "serving from SQLite", ", ".join(missing),
+        )
+        return AutoPromotionResult(
+            False, False, f"{', '.join(missing)} will not import",
+        )
+
+    try:
+        from superlocalmemory.core.scale_engine import ScaleEngineManager
+    except Exception as exc:  # noqa: BLE001
+        return AutoPromotionResult(False, False, f"scale engine unavailable: {exc}")
+
+    try:
+        manager = ScaleEngineManager(config)
+    except Exception as exc:  # noqa: BLE001
+        return AutoPromotionResult(False, False, f"could not open the store: {exc}")
+
+    try:
+        status = manager.status()
+    except Exception as exc:  # noqa: BLE001
+        return AutoPromotionResult(False, False, f"could not read the state: {exc}")
+
+    if status.get("migration_repair_required"):
+        # An interrupted promotion has its own recovery path, and starting a
+        # fresh one on top of it would be building over a half-finished move.
+        logger.warning(
+            "a previous promotion did not finish; leaving it for repair rather "
+            "than starting another",
+        )
+        return AutoPromotionResult(False, False, "a previous promotion needs repair")
+
+    try:
+        # Resume a stage that is already built rather than building another.
+        # Only "promoted" used to stop this, so a start that prepared and then
+        # failed to verify left the state at "prepared" and the NEXT start
+        # built a second stage — and the one after that a third, with the
+        # staging directory growing every time.
+        stage_id = _resumable_stage(status)
+        if stage_id:
+            logger.info("resuming the projection already staged as %s", stage_id)
+        else:
+            prepared = manager.prepare()
+            stage_id = str(prepared.get("stage_id", ""))
+        verified = manager.verify(stage_id)
+        if str(verified.get("state")) != "verified":
+            logger.warning(
+                "projection did not match the canonical store; not promoting",
+            )
+            return AutoPromotionResult(
+                True, False, "the projection did not match the store", stage_id,
+            )
+        promoted = manager.promote(stage_id)
+    except Exception as exc:  # noqa: BLE001 - reported, never fatal
+        logger.warning(
+            "could not move onto the graph and vector backends (%s); serving "
+            "from SQLite", exc,
+        )
+        return AutoPromotionResult(True, False, str(exc))
+
+    logger.info(
+        "graph and vector backends promoted (stage %s); they serve after the "
+        "next start", promoted.get("stage_id", ""),
+    )
+    return AutoPromotionResult(
+        True,
+        True,
+        "promoted",
+        str(promoted.get("stage_id", "")),
+        bool(promoted.get("restart_required", True)),
+    )

package/src/superlocalmemory/core/scale_engine.py

@@ -555,10 +555,24 @@ class ScaleEngineManager:
         projects them: dedup entity IDs per fact, drop empties. This bridge is
         what lets Cozo map a query seed into the fact graph; if its import
         silently fails, count parity on entities/edges/vectors still passes but
-        entity recall returns empty — so it must be verified explicitly."""
+        entity recall returns empty — so it must be verified explicitly.
+
+        Withheld and archived facts are excluded, because the import excludes
+        them. On this store that is not a rounding difference: 1,291 withheld
+        facts hold 379,591 of the 395,524 bridge rows — about 294 entity
+        references each, against roughly 4 for an ordinary memory. That is the
+        "pooled entity list" the retrieval channel refuses to let into its entity
+        map, and it is why a withheld row out-ranks a real one. Counting them
+        here made the parity check demand a projection the import must not build,
+        and promotion failed on it."""
+        from superlocalmemory.storage.database import (
+            visible_fact_clause_for_connection,
+        )
+
         total = 0
         for (raw,) in conn.execute(
-            "SELECT canonical_entities_json FROM atomic_facts WHERE profile_id=?",
+            "SELECT canonical_entities_json FROM atomic_facts WHERE profile_id=?"
+            + visible_fact_clause_for_connection(conn),
             (self.profile_id,),
         ):
             try:

package/src/superlocalmemory/core/score_contract.py

@@ -27,7 +27,27 @@ def _bounded(value: object, default: float = 0.0) -> float:
 
 
 def finalize_score_contract(response: RecallResponse) -> RecallResponse:
-    """Finalize aliases, rank positions, and response abstention metadata."""
+    """Finalize aliases, rank positions, and response abstention metadata.
+
+    ``score`` IS NOT THE ORDERING KEY, and a caller that sorts by it will get a
+    different answer than the one this returns. That is deliberate, and it is
+    load-bearing:
+
+    * ``score`` answers "how well does this match the query" — a property of the
+      result and the query alone.
+    * ``rank_position``, assigned here from the order of the list, answers "what
+      is recommended, and in what order". Ranking also weighs whether a memory
+      has helped before and whether the session was just looking at it, neither
+      of which is a property of the query.
+
+    So the two can disagree, and the top result can legitimately carry a lower
+    ``score`` than the one beneath it. Consumers should present the list in the
+    order given, or sort by ``rank_position``; ``ranking_score`` carries the
+    internal utility for anyone who needs to re-derive it.
+
+    This ordering is NOT recomputed here. Re-sorting by ``score`` at this point
+    would silently undo every ranking pass that ran before it.
+    """
     for position, result in enumerate(response.results or (), start=1):
         relevance = _bounded(
             result.relevance_score

package/src/superlocalmemory/core/session_identity.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Which session ids name a conversation, and which were invented for one call.
+
+TWO DIFFERENT JOBS, ONE STRING
+------------------------------
+A session id is used for two unrelated things:
+
+1. **Bookkeeping.** A recall needs *some* handle so a downstream reference can be
+   traced back to it. Any unique string does. When a caller supplies none, the
+   HTTP and MCP fronts invent one — ``http:<milliseconds>`` per request, or
+   ``mcp:<agent_id>`` per client.
+
+2. **Continuity.** The working set carries what a conversation was recently
+   shown into its next turn. That needs an id that actually identifies a
+   conversation, because it decides what gets ranked higher.
+
+An invented id is fine for the first and wrong for the second, and the two are
+wrong in opposite directions:
+
+* ``http:<ms>`` is unique per request, so every dashboard search registered a
+  new working set. Enough of them and the registry hits its cap and evicts the
+  least-recently-touched entry — which is a real conversation sitting idle
+  between turns. The next turn of that conversation is cold, with no error.
+* ``mcp:<agent_id>`` is SHARED by every client that did not send an id, so two
+  unrelated clients pooled one seven-slot set and promoted each other's
+  memories.
+
+Neither mattered while the parameter was ignored, which it was until continuity
+was built on it. That is what made this easy to miss.
+
+ONE DEFINITION
+--------------
+Both the creators and the reader use this module, so a new front cannot invent a
+third prefix that continuity silently accepts. Two lists of the same thing is how
+one ends up wrong.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "SYNTHETIC_PREFIXES",
+    "is_conversation",
+    "synthetic_session_id",
+]
+
+#: Prefixes marking an id that a front invented rather than received.
+#:
+#: The colon is deliberate: a real client id is a uuid or a hex string and does
+#: not contain one, so a genuine id cannot be mistaken for an invented one.
+SYNTHETIC_PREFIXES: tuple[str, ...] = ("http:", "mcp:", "cli:", "probe:")
+
+
+def synthetic_session_id(kind: str, discriminator: str = "") -> str:
+    """Build an id for bookkeeping that continuity will correctly ignore.
+
+    ``kind`` names the front that invented it, so a log line says where an
+    unattributed recall came from.
+    """
+    prefix = kind if kind.endswith(":") else f"{kind}:"
+    return f"{prefix}{discriminator}"
+
+
+def is_conversation(
+    session_id: str | None, profile_id: str | None = None,
+) -> bool:
+    """Whether this pair identifies a conversation across turns.
+
+    False for an empty id and for anything a front invented. Continuity engages
+    only when this is True, so the default for an unidentified caller is the
+    behaviour that existed before continuity: every recall starts cold.
+
+    ``profile_id`` is checked too when given. The working set is keyed on both,
+    and an empty profile is not a profile: ``None`` and ``""`` would normalise to
+    the same key, so two unidentified callers sharing a session id would share
+    one set. Every caller in this codebase resolves a real profile before
+    reaching here, which is exactly why the aliasing would go unnoticed if it
+    ever stopped being true.
+    """
+    if not session_id:
+        return False
+    if profile_id is not None and not profile_id:
+        return False
+    return not session_id.startswith(SYNTHETIC_PREFIXES)

package/src/superlocalmemory/core/status_contract.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""What "status" means, in one place, for every surface that answers it.
+
+Three surfaces answered the same question — MCP ``get_status``, the HTTP
+dashboard and ``slm status`` — and returned three different field sets. Two of
+them omitted the graph counts, which are the numbers that say whether the graph
+is healthy; one omitted the version, which is the first thing anyone asks for
+in a bug report. Each carried its own copy of the same three COUNT queries.
+
+This module holds the agreed field set and the queries behind it. A surface may
+add fields of its own — the dashboard needs a display name for the mode, the
+daemon needs its own pid and uptime — but it may not be missing one of these.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+#: Fields every status surface emits. A surface with extra fields is fine; a
+#: surface missing one of these is a defect, and a test asserts it.
+CANONICAL_STATUS_FIELDS: tuple[str, ...] = (
+    "mode",
+    "provider",
+    "profile",
+    "base_dir",
+    "db_path",
+    "db_size_mb",
+    "fact_count",
+    "entity_count",
+    "edge_count",
+    "profile_generation",
+    "version",
+    "projection_queue_depth",
+)
+
+#: The counts, and the one query each comes from. Profile-scoped without
+#: exception — a count that ignores the active profile is a wrong answer on any
+#: machine with more than one workspace.
+#:
+#: ``fact_count`` is completed at call time with the visibility predicate: it is
+#: the number an owner reads as "how much do I remember", so soft-deleted and
+#: withheld rows must not be in it. Counting them raw is how the dashboard came
+#: to report 5,317 against 4,018 real memories on the author's store.
+COUNT_QUERIES: dict[str, str] = {
+    "fact_count": "SELECT COUNT(*) FROM atomic_facts WHERE profile_id = ?",
+    "entity_count": "SELECT COUNT(*) FROM canonical_entities WHERE profile_id = ?",
+    "edge_count": "SELECT COUNT(*) FROM graph_edges WHERE profile_id = ?",
+}
+
+
+def counts_from_sqlite(conn: sqlite3.Connection, profile_id: str) -> dict[str, int]:
+    """Every count on the contract, from one connection.
+
+    A table that is not there yet — a store mid-migration, or one built before
+    the graph existed — reports zero rather than failing the whole status call.
+    A status endpoint that raises tells an operator nothing.
+    """
+    from superlocalmemory.storage.database import visible_fact_clause_for_connection
+
+    counts: dict[str, int] = {}
+    for field, sql in COUNT_QUERIES.items():
+        if field == "fact_count":
+            sql += visible_fact_clause_for_connection(conn)
+        try:
+            row = conn.execute(sql, (profile_id,)).fetchone()
+            counts[field] = int(row[0]) if row else 0
+        except sqlite3.Error:
+            counts[field] = 0
+    return counts
+
+
+def projection_queue_depth(conn: sqlite3.Connection) -> int:
+    """Facts stored but not yet carried into the graph and vector projections.
+
+    Zero is the healthy steady state. A number that does not fall means the
+    projections have stopped keeping up with the store, which is the one failure
+    that would otherwise be invisible: the memory is safely in SQLite, so
+    nothing errors, and it is simply missing from the graph until someone
+    notices recall got worse.
+
+    Deliberately NOT profile-scoped. The worker is one queue for the whole
+    store, and a projection stalled on another workspace's facts is still a
+    stalled projection. Scoping it per profile would let a status page report
+    zero while the drain was wedged.
+    """
+    from superlocalmemory.storage.projection_outbox import DEPTH_SQL
+
+    try:
+        row = conn.execute(DEPTH_SQL).fetchone()
+    except sqlite3.Error:
+        # A store that predates the queue has nothing pending by definition.
+        return 0
+    return int(row[0]) if row else 0
+
+
+def store_size_mb(db_path: Path | str | None) -> float:
+    """Size of the store on disk, or 0.0 when there is nothing to measure."""
+    if not db_path:
+        return 0.0
+    path = Path(db_path)
+    try:
+        return round(path.stat().st_size / (1024 * 1024), 2)
+    except OSError:
+        return 0.0

package/src/superlocalmemory/core/worker_pool.py

@@ -78,10 +78,10 @@ class WorkerPool:
     ) -> dict:
         """Run recall in worker subprocess. Returns result dict.
 
-        S9-DASH-02: ``session_id`` threads through to ``engine.recall``
-        so the outcome-queue gets a pending_outcomes row for this
-        recall. Without it, hook-based signals have no outcome to
-        attach to.
+        ``session_id`` threads through to ``engine.recall``, where it gives
+        the recall continuity with earlier turns of the same session. It does
+        NOT create an outcome row — this docstring claimed one for several
+        releases and none was ever written.
 
         v3.6.15 multi-scope: ``include_global``/``include_shared`` are forwarded
         to the worker (and on to ``engine.recall``). ``None`` is sent verbatim