superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/core/session_identity.py

@@ -1,98 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-
-"""Which session ids name a conversation, and which were invented for one call.
-
-TWO DIFFERENT JOBS, ONE STRING
-------------------------------
-A session id is used for two unrelated things:
-
-1. **Bookkeeping.** A recall needs *some* handle so a downstream reference can be
-   traced back to it. Any unique string does. When a caller supplies none, the
-   HTTP and MCP fronts invent one — ``http:<milliseconds>`` per request, or
-   ``mcp:<agent_id>`` per client.
-
-2. **Continuity.** The working set carries what a conversation was recently
-   shown into its next turn. That needs an id that actually identifies a
-   conversation, because it decides what gets ranked higher.
-
-An invented id is fine for the first and wrong for the second, and the two are
-wrong in opposite directions:
-
-* ``http:<ms>`` is unique per request, so every dashboard search registered a
-  new working set. Enough of them and the registry hits its cap and evicts the
-  least-recently-touched entry — which is a real conversation sitting idle
-  between turns. The next turn of that conversation is cold, with no error.
-* ``mcp:<agent_id>`` is SHARED by every client that did not send an id, so two
-  unrelated clients pooled one seven-slot set and promoted each other's
-  memories.
-
-Neither mattered while the parameter was ignored, which it was until continuity
-was built on it. That is what made this easy to miss.
-
-ONE DEFINITION
---------------
-Both the creators and the reader use this module, so a new front cannot invent a
-third prefix that continuity silently accepts. Two lists of the same thing is how
-one ends up wrong.
-"""
-
-from __future__ import annotations
-
-__all__ = [
-    "SYNTHETIC_PREFIXES",
-    "is_conversation",
-    "synthetic_session_id",
-]
-
-#: Prefixes marking an id that a front invented rather than received.
-#:
-#: The colon is deliberate: a real client id is a uuid or a hex string and does
-#: not contain one, so a genuine id cannot be mistaken for an invented one.
-#: ``engine:`` names the daemon process itself, not a caller.  It is shared by
-#: every client that reaches one daemon and changes on every restart, so it is
-#: synthetic in exactly the way this module exists to catch: it was minted by
-#: hand in ``core/engine.py`` rather than through ``synthetic_session_id``, and
-#: so passed ``is_conversation`` for as long as it existed.  While it did, every
-#: recall that named no session filed its outcome under a process id that no
-#: tool event could ever carry, and the reward pipeline had nothing to join on.
-#: ``agent:`` and ``api:`` name the calling agent and the workspace, and are
-#: shared by every request from either — ``agent:mcp_client`` alone held 24
-#: outcomes from unrelated callers. Both were hand-minted in
-#: ``server/routes/v3_api.py``, which is why neither was listed here.
-SYNTHETIC_PREFIXES: tuple[str, ...] = (
-    "http:", "mcp:", "cli:", "probe:", "engine:", "agent:", "api:",
-)
-
-
-def synthetic_session_id(kind: str, discriminator: str = "") -> str:
-    """Build an id for bookkeeping that continuity will correctly ignore.
-
-    ``kind`` names the front that invented it, so a log line says where an
-    unattributed recall came from.
-    """
-    prefix = kind if kind.endswith(":") else f"{kind}:"
-    return f"{prefix}{discriminator}"
-
-
-def is_conversation(
-    session_id: str | None, profile_id: str | None = None,
-) -> bool:
-    """Whether this pair identifies a conversation across turns.
-
-    False for an empty id and for anything a front invented. Continuity engages
-    only when this is True, so the default for an unidentified caller is the
-    behaviour that existed before continuity: every recall starts cold.
-
-    ``profile_id`` is checked too when given. The working set is keyed on both,
-    and an empty profile is not a profile: ``None`` and ``""`` would normalise to
-    the same key, so two unidentified callers sharing a session id would share
-    one set. Every caller in this codebase resolves a real profile before
-    reaching here, which is exactly why the aliasing would go unnoticed if it
-    ever stopped being true.
-    """
-    if not session_id:
-        return False
-    if profile_id is not None and not profile_id:
-        return False
-    return not session_id.startswith(SYNTHETIC_PREFIXES)

package/src/superlocalmemory/core/shadow_router.py

@@ -1,319 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.4.22 — Stage 8 SB-1 / Track A.3 (LLD-10)
-
-"""Daemon-resident recall-time A/B router for LLD-10 shadow + rollback.
-
-This module is the single seam that wires ``ShadowTest`` (pre-promotion)
-and ``ModelRollback`` (post-promotion) into the live recall path. Before
-this module existed (pre-v3.4.22 Stage 8 SB-1) both classes were
-defined + unit-tested but had zero production callers in ``src/``.
-
-Design constraints:
-
-  * Process-local state only — one ``ShadowRouter`` instance per
-    ``(memory_db, learning_db, profile_id)`` tuple, held by the
-    daemon's in-process singleton cache.
-  * Deterministic A/B routing — ``route_query(qid)`` uses
-    ``sha256(install_token + qid)`` so an attacker who controls
-    ``qid`` still cannot bias the split without reading the install
-    token on disk (closes skeptic H-02 + H-03).
-  * Promotion and rollback are DELEGATED to the canonical helpers in
-    ``learning/consolidation_worker`` and ``learning/model_rollback``.
-    The router never writes to ``learning_model_state`` directly.
-  * Fail-soft: any exception during recall-time ingestion is logged
-    and swallowed — we must not break the user's recall path.
-
-References:
-  - LLD-00 §8    — two-phase shadow + auto-rollback.
-  - LLD-10 §4.1  — deterministic A/B routing.
-  - LLD-10 §5    — atomic BEGIN IMMEDIATE promotion + rollback.
-  - Stage 8 SB-1 — architect S8-ARC-C1, skeptic C-01/C-02/H-07.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import logging
-import threading
-from typing import Final
-
-from superlocalmemory.core.security_primitives import ensure_install_token
-from superlocalmemory.learning.model_rollback import ModelRollback
-from superlocalmemory.learning.shadow_test import ShadowTest
-
-logger = logging.getLogger(__name__)
-
-
-# ---------------------------------------------------------------------------
-# Arm vocabulary — externalised so hooks can grep-check.
-# ---------------------------------------------------------------------------
-
-ARM_BASELINE: Final[str] = "baseline"
-ARM_CANDIDATE: Final[str] = "candidate"
-
-
-class ShadowRouter:
-    """Per-profile A/B router + post-promotion watcher.
-
-    Not thread-safe for write state (`_shadow`, `_rollback`) — the daemon
-    calls ``on_recall_settled`` from a single recall-settled worker queue.
-    Reads (``route_query``) are safe under concurrency.
-    """
-
-    def __init__(
-        self,
-        *,
-        memory_db: str,
-        learning_db: str,
-        profile_id: str,
-    ) -> None:
-        self._memory_db = str(memory_db)
-        self._learning_db = str(learning_db)
-        self._profile_id = str(profile_id)
-        # Shadow pre-promotion accumulator (created lazily when a
-        # candidate is persisted).
-        self._shadow: ShadowTest | None = None
-        self._candidate_id: int | None = None
-        # S9-SKEP-11: pin the routing token at router creation.
-        # ``install_token`` rotation is a documented user action (SEC-M07
-        # ``slm escape-hatch rotate-token``) but must not flip the arm
-        # assignment of every in-flight query_id — that would silently
-        # mix baseline and candidate observations for the same qid and
-        # break the paired-t assumption. We snapshot the token once at
-        # init; subsequent rotations are picked up only when a new
-        # router is created (i.e. at daemon restart / candidate rotate).
-        try:
-            self._routing_token = ensure_install_token()
-        except Exception:  # pragma: no cover — defensive
-            self._routing_token = ""
-        # Rollback post-promotion watcher (created when arm_post_promotion
-        # is called after a promote).
-        self._rollback: ModelRollback | None = None
-        # S9-W4 H-ARC-01: re-attach an existing candidate across daemon
-        # restart. The prior implementation lost the candidate_id on
-        # process exit — a 6-hour shadow test that had persisted a
-        # candidate an hour in would restart the observation window
-        # from zero on the next boot, extending the test indefinitely.
-        # Observations themselves do NOT survive restart (they live
-        # only in ``ShadowTest._active/_candidate`` lists and would
-        # require a new schema to persist durably); re-attaching the
-        # candidate_id is the minimum that keeps the A/B loop from
-        # silently stalling.
-        self._reattach_existing_candidate()
-
-    def _reattach_existing_candidate(self) -> None:
-        """H-ARC-01: look up any ``is_candidate=1`` row for this profile
-        and restore ``self._candidate_id`` + a fresh ``ShadowTest``.
-
-        Fail-soft — on any DB error we leave the router empty and the
-        normal lazy-init path takes over as candidates land.
-        """
-        try:
-            import sqlite3 as _sq
-            conn = _sq.connect(self._learning_db, timeout=2.0)
-            try:
-                row = conn.execute(
-                    "SELECT id FROM learning_model_state "
-                    "WHERE profile_id = ? AND is_candidate = 1 "
-                    "LIMIT 1",
-                    (self._profile_id,),
-                ).fetchone()
-            finally:
-                conn.close()
-        except Exception:
-            return  # learning.db or table missing — normal cold start.
-        if row and row[0] is not None:
-            cid = int(row[0])
-            self._candidate_id = cid
-            # Pass learning_db so ShadowTest reloads persisted paired
-            # observations from ``shadow_observations`` (M012).
-            self._shadow = ShadowTest(
-                profile_id=self._profile_id,
-                candidate_model_id=str(cid),
-                learning_db=self._learning_db,
-            )
-
-    # ------------------------------------------------------------------
-    # Routing
-    # ------------------------------------------------------------------
-
-    def route_query(self, query_id: str) -> str:
-        """Return the arm for ``query_id``: ``'baseline'`` or ``'candidate'``.
-
-        Deterministic per ``(install_token, query_id)`` — survives
-        daemon restart. Install-token dependence closes the
-        attacker-picks-query-id bias vector (skeptic H-02).
-
-        S9-SKEP-11: uses ``self._routing_token`` (pinned at router
-        init) instead of a fresh ``ensure_install_token()`` per call,
-        so mid-test token rotation does not silently flip the arm of
-        every subsequent qid and mix the paired-t arms.
-        """
-        digest = hashlib.sha256(
-            (self._routing_token + str(query_id)).encode("utf-8"),
-        ).hexdigest()[:8]
-        return ARM_CANDIDATE if int(digest, 16) % 2 == 1 else ARM_BASELINE
-
-    # ------------------------------------------------------------------
-    # Pre-promotion accumulator
-    # ------------------------------------------------------------------
-
-    def attach_candidate(self, candidate_id: int) -> None:
-        """Called after ``_persist_candidate`` writes a fresh candidate row.
-        Creates a new ``ShadowTest`` to collect paired recall results.
-        """
-        self._candidate_id = int(candidate_id)
-        # Pass learning_db so new paired observations persist to
-        # ``shadow_observations`` (M012) for restart durability.
-        self._shadow = ShadowTest(
-            profile_id=self._profile_id,
-            candidate_model_id=str(candidate_id),
-            learning_db=self._learning_db,
-        )
-
-    # ------------------------------------------------------------------
-    # Post-promotion watcher
-    # ------------------------------------------------------------------
-
-    def arm_post_promotion_watch(self, *, baseline_ndcg: float) -> None:
-        """Install a fresh ``ModelRollback`` observer for the 200-recall
-        post-promotion watch window."""
-        self._rollback = ModelRollback(
-            learning_db_path=self._learning_db,
-            profile_id=self._profile_id,
-            baseline_ndcg=float(baseline_ndcg),
-        )
-
-    # ------------------------------------------------------------------
-    # Recall-settled ingestion
-    # ------------------------------------------------------------------
-
-    def on_recall_settled(
-        self,
-        *,
-        query_id: str,
-        arm: str,
-        ndcg_at_10: float,
-    ) -> None:
-        """Feed one settled recall into whichever phase is active.
-
-        Precedence:
-          1. If a ShadowTest is active → record paired observation; on
-             ``decide() == 'promote'`` fire the atomic promotion.
-          2. If a ModelRollback watch is active → record observation;
-             on ``should_rollback() is True`` fire execute_rollback.
-
-        Both phases may run for the same profile briefly if a promote
-        fires mid-batch; that is intentional — the first watch window
-        starts immediately on promotion.
-        """
-        try:
-            if self._shadow is not None and self._candidate_id is not None:
-                # ShadowTest expects arm='active'|'candidate'. Our router
-                # uses 'baseline' as the externally-visible arm name.
-                st_arm = "active" if arm == ARM_BASELINE else "candidate"
-                self._shadow.record_recall_pair(
-                    query_id=str(query_id), arm=st_arm,
-                    ndcg_at_10=float(ndcg_at_10),
-                )
-                decision, _stats = self._shadow.decide()
-                if decision == "promote":
-                    self._fire_promotion()
-                elif decision == "reject":
-                    # Release the candidate reservation — a future
-                    # _run_shadow_cycle may insert a new candidate.
-                    self._shadow = None
-                    self._candidate_id = None
-
-            if self._rollback is not None:
-                self._rollback.record_post_promotion(
-                    query_id=str(query_id), ndcg_at_10=float(ndcg_at_10),
-                )
-                if self._rollback.should_rollback():
-                    self._fire_rollback()
-        except Exception as exc:  # pragma: no cover — defensive
-            logger.debug("shadow_router on_recall_settled error: %s", exc)
-
-    # ------------------------------------------------------------------
-    # Internal: promotion + rollback triggers
-    # ------------------------------------------------------------------
-
-    def _fire_promotion(self) -> None:
-        """Call the canonical ``_promote_candidate`` helper and reset
-        the shadow state so a new A/B cycle can start next retrain."""
-        if self._candidate_id is None:
-            return
-        try:
-            from superlocalmemory.learning.consolidation_worker import (
-                _promote_candidate,
-            )
-            _promote_candidate(
-                self._learning_db,
-                profile_id=self._profile_id,
-                candidate_id=int(self._candidate_id),
-            )
-        except Exception as exc:  # pragma: no cover — defensive
-            logger.warning("shadow_router promotion failed: %s", exc)
-        finally:
-            self._shadow = None
-            self._candidate_id = None
-
-    def _fire_rollback(self) -> None:
-        """Call ``ModelRollback.execute_rollback`` and clear the watcher."""
-        try:
-            if self._rollback is not None:
-                self._rollback.execute_rollback(
-                    reason="watch_window_regression",
-                )
-        except Exception as exc:  # pragma: no cover — defensive
-            logger.warning("shadow_router rollback failed: %s", exc)
-        finally:
-            self._rollback = None
-
-
-# ---------------------------------------------------------------------------
-# Process-local singleton cache — one ShadowRouter per (learning_db, profile).
-# Thread-safe; keyed by a stable tuple so repeated factory calls reuse state.
-# ---------------------------------------------------------------------------
-
-
-_CACHE: dict[tuple[str, str, str], ShadowRouter] = {}
-_CACHE_LOCK = threading.Lock()
-
-
-def get_shadow_router(
-    *,
-    memory_db: str,
-    learning_db: str,
-    profile_id: str,
-) -> ShadowRouter:
-    """Factory — returns the process-local ``ShadowRouter`` for
-    ``(memory_db, learning_db, profile_id)``, creating one on first call.
-    """
-    key = (str(memory_db), str(learning_db), str(profile_id))
-    with _CACHE_LOCK:
-        router = _CACHE.get(key)
-        if router is None:
-            router = ShadowRouter(
-                memory_db=memory_db,
-                learning_db=learning_db,
-                profile_id=profile_id,
-            )
-            _CACHE[key] = router
-        return router
-
-
-def reset_for_testing() -> None:
-    """Clear the singleton cache — tests only."""
-    with _CACHE_LOCK:
-        _CACHE.clear()
-
-
-__all__ = (
-    "ARM_BASELINE",
-    "ARM_CANDIDATE",
-    "ShadowRouter",
-    "get_shadow_router",
-    "reset_for_testing",
-)

package/src/superlocalmemory/core/slm_disabled.py

@@ -1,85 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory v3.4.22 — Stage 8 SB-5
-
-"""Global kill-switch for SLM.
-
-Two mechanisms, either disables the entire system cheaply:
-
-1. File-marker: ``~/.superlocalmemory/.disabled`` — persistent across
-   reboots, survives daemon restarts, written by ``slm disable``.
-2. Environment variable: ``SLM_DISABLE=1`` — per-process, useful for
-   CI, sandboxes, or "just for this shell" overrides.
-
-Every hot-path entry point (hooks, MCP tools, recall pipeline, daemon
-lifespan) calls :func:`is_disabled` first. Returns ``True`` ⇒ exit
-quietly, no side effects.
-
-Backward-compat: unset env + missing marker ⇒ ``False`` ⇒ normal
-behaviour. Zero impact on the 18k live users who never touch it.
-"""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-
-
-_MARKER_NAME = ".disabled"
-_ENV_NAME = "SLM_DISABLE"
-
-
-def _slm_home() -> Path:
-    """Return the canonical SLM runtime-state directory."""
-    from superlocalmemory.infra.data_root import canonical_data_root
-    return canonical_data_root()
-
-
-def marker_path() -> Path:
-    """Where the persistent ``.disabled`` marker lives."""
-    return _slm_home() / _MARKER_NAME
-
-
-def is_disabled() -> bool:
-    """Return True iff SLM should no-op everything.
-
-    Precedence: env var first (cheapest check), then file marker. Any
-    non-empty, non-"0", non-"false" value in the env counts as disabled.
-    """
-    env = os.environ.get(_ENV_NAME, "").strip().lower()
-    if env and env not in ("0", "false", "no", "off"):
-        return True
-    try:
-        return marker_path().exists()
-    except OSError:  # pragma: no cover — defensive against FS errors
-        return False
-
-
-def write_marker(reason: str = "") -> Path:
-    """Create the disabled marker. Returns the path."""
-    home = _slm_home()
-    home.mkdir(parents=True, exist_ok=True)
-    path = home / _MARKER_NAME
-    payload = "disabled"
-    if reason:
-        payload = f"disabled: {reason}\n"
-    path.write_text(payload, encoding="utf-8")
-    return path
-
-
-def remove_marker() -> bool:
-    """Remove the disabled marker. Returns True if removed, False if absent."""
-    path = marker_path()
-    try:
-        path.unlink()
-        return True
-    except FileNotFoundError:
-        return False
-
-
-__all__ = (
-    "is_disabled",
-    "write_marker",
-    "remove_marker",
-    "marker_path",
-)

package/src/superlocalmemory/core/status_contract.py

@@ -1,108 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""What "status" means, in one place, for every surface that answers it.
-
-Three surfaces answered the same question — MCP ``get_status``, the HTTP
-dashboard and ``slm status`` — and returned three different field sets. Two of
-them omitted the graph counts, which are the numbers that say whether the graph
-is healthy; one omitted the version, which is the first thing anyone asks for
-in a bug report. Each carried its own copy of the same three COUNT queries.
-
-This module holds the agreed field set and the queries behind it. A surface may
-add fields of its own — the dashboard needs a display name for the mode, the
-daemon needs its own pid and uptime — but it may not be missing one of these.
-"""
-
-from __future__ import annotations
-
-import sqlite3
-from pathlib import Path
-
-#: Fields every status surface emits. A surface with extra fields is fine; a
-#: surface missing one of these is a defect, and a test asserts it.
-CANONICAL_STATUS_FIELDS: tuple[str, ...] = (
-    "mode",
-    "provider",
-    "profile",
-    "base_dir",
-    "db_path",
-    "db_size_mb",
-    "fact_count",
-    "entity_count",
-    "edge_count",
-    "profile_generation",
-    "version",
-    "projection_queue_depth",
-)
-
-#: The counts, and the one query each comes from. Profile-scoped without
-#: exception — a count that ignores the active profile is a wrong answer on any
-#: machine with more than one workspace.
-#:
-#: ``fact_count`` is completed at call time with the visibility predicate: it is
-#: the number an owner reads as "how much do I remember", so soft-deleted and
-#: withheld rows must not be in it. Counting them raw is how the dashboard came
-#: to report 5,317 against 4,018 real memories on the author's store.
-COUNT_QUERIES: dict[str, str] = {
-    "fact_count": "SELECT COUNT(*) FROM atomic_facts WHERE profile_id = ?",
-    "entity_count": "SELECT COUNT(*) FROM canonical_entities WHERE profile_id = ?",
-    "edge_count": "SELECT COUNT(*) FROM graph_edges WHERE profile_id = ?",
-}
-
-
-def counts_from_sqlite(conn: sqlite3.Connection, profile_id: str) -> dict[str, int]:
-    """Every count on the contract, from one connection.
-
-    A table that is not there yet — a store mid-migration, or one built before
-    the graph existed — reports zero rather than failing the whole status call.
-    A status endpoint that raises tells an operator nothing.
-    """
-    from superlocalmemory.storage.database import visible_fact_clause_for_connection
-
-    counts: dict[str, int] = {}
-    for field, sql in COUNT_QUERIES.items():
-        if field == "fact_count":
-            sql += visible_fact_clause_for_connection(conn)
-        try:
-            row = conn.execute(sql, (profile_id,)).fetchone()
-            counts[field] = int(row[0]) if row else 0
-        except sqlite3.Error:
-            counts[field] = 0
-    return counts
-
-
-def projection_queue_depth(conn: sqlite3.Connection) -> int:
-    """Facts stored but not yet carried into the graph and vector projections.
-
-    Zero is the healthy steady state. A number that does not fall means the
-    projections have stopped keeping up with the store, which is the one failure
-    that would otherwise be invisible: the memory is safely in SQLite, so
-    nothing errors, and it is simply missing from the graph until someone
-    notices recall got worse.
-
-    Deliberately NOT profile-scoped. The worker is one queue for the whole
-    store, and a projection stalled on another workspace's facts is still a
-    stalled projection. Scoping it per profile would let a status page report
-    zero while the drain was wedged.
-    """
-    from superlocalmemory.storage.projection_outbox import DEPTH_SQL
-
-    try:
-        row = conn.execute(DEPTH_SQL).fetchone()
-    except sqlite3.Error:
-        # A store that predates the queue has nothing pending by definition.
-        return 0
-    return int(row[0]) if row else 0
-
-
-def store_size_mb(db_path: Path | str | None) -> float:
-    """Size of the store on disk, or 0.0 when there is nothing to measure."""
-    if not db_path:
-        return 0.0
-    path = Path(db_path)
-    try:
-        return round(path.stat().st_size / (1024 * 1024), 2)
-    except OSError:
-        return 0.0