superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/server/rbac_enforce.py

@@ -1,194 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 — RBAC / teams (C3)
-
-"""RBAC enforcement boundary for HTTP routes.
-
-This is the single place that turns "who is calling" + "what are they trying to
-do" into an allow/deny decision, on top of the existing machine-auth layer
-(write_identity). It is deliberately small so every mutation route calls the
-same code path — the research warning was explicit: an RBAC layer that is
-defined but not consistently called is worse than none.
-
-Principal model
----------------
-* **user**  — a logged-in dashboard user (valid session token). Always enforced
-  against their role on the active profile.
-* **owner** — the machine operator (already proved machine auth via
-  write_identity; no user session). In personal mode the owner bypasses RBAC
-  (all permissions). When the org turns on *require_login* (company mode) the
-  owner bypass is disabled and a user session is mandatory.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-from fastapi import HTTPException, Request
-
-from superlocalmemory.access.rbac import Permission, Role, permissions_for_role
-import logging
-
-logger = logging.getLogger(__name__)
-
-_SESSION_HEADER = "X-SLM-User-Session"
-_SESSION_COOKIE = "slm_session"
-
-OWNER_PRINCIPAL = {
-    "kind": "owner",
-    "user_id": "owner",
-    "username": "owner",
-    "display_name": "Machine Owner",
-}
-
-
-def get_rbac_engine(app_state: Any) -> Any | None:
-    return getattr(app_state, "rbac", None)
-
-
-def _session_token(request: Request) -> str:
-    tok = request.headers.get(_SESSION_HEADER, "")
-    if tok:
-        return tok
-    try:
-        return request.cookies.get(_SESSION_COOKIE, "") or ""
-    except Exception:
-        return ""
-
-
-def resolve_principal(request: Request) -> dict:
-    """Resolve the caller to a user (valid session) or the machine owner."""
-    rbac = get_rbac_engine(request.app.state)
-    token = _session_token(request)
-    if rbac is not None and token:
-        user = rbac.resolve_session(token)
-        if user:
-            return {"kind": "user", **user}
-    return dict(OWNER_PRINCIPAL)
-
-
-def _active_profile() -> str:
-    from superlocalmemory.server.routes.helpers import get_active_profile
-
-    return get_active_profile()
-
-
-def require_permission(
-    request: Request,
-    permission: Permission,
-    *,
-    profile: str | None = None,
-) -> dict:
-    """Authorize ``permission`` on ``profile`` (default: active profile).
-
-    Returns the principal on success. Raises 401 when a login is required but
-    absent, or 403 when the user's role does not grant the permission.
-    """
-    rbac = get_rbac_engine(request.app.state)
-    principal = resolve_principal(request)
-    require_login = bool(rbac is not None and rbac.require_login())
-    prof = profile or _active_profile()
-
-    if principal["kind"] == "owner":
-        # The machine operator is root — they always retain MANAGE (they have
-        # shell access to the box regardless), so company mode can never lock
-        # administration out of the dashboard. require_login only gates the
-        # owner's DATA operations, forcing per-user login for read/write/etc.
-        if require_login and permission != Permission.MANAGE:
-            raise HTTPException(
-                401,
-                detail="Login required: this workspace enforces per-user access.",
-            )
-        return principal  # personal mode — operator is owner
-
-    # Logged-in user: always enforced against their role.
-    if rbac is not None and rbac.has_permission(principal["user_id"], prof, permission):
-        return principal
-    raise HTTPException(
-        403,
-        detail=(
-            f"Your role does not allow '{permission.value}' on this workspace."
-        ),
-    )
-
-
-def require_manage(request: Request, *, profile: str | None = None) -> dict:
-    """Guard for user/role administration (MANAGE permission)."""
-    return require_permission(request, Permission.MANAGE, profile=profile)
-
-
-def resolve_actor_roles(request: Request, *, profile: str | None = None):
-    """Resolve the caller to concrete ActorContext roles (server-derived).
-
-    The machine operator (owner) is root. A logged-in user is mapped from their
-    persisted RBAC role on ``profile``. This must be called only after
-    ``require_permission`` has already authorized the operation, so the returned
-    role always includes the permission the caller was admitted with.
-    """
-    from superlocalmemory.core.actor_context import ActorRole
-
-    principal = resolve_principal(request)
-    if principal.get("kind") == "owner":
-        return frozenset({ActorRole.OWNER})
-    rbac = get_rbac_engine(request.app.state)
-    role = None
-    if rbac is not None:
-        try:
-            role = rbac.get_role(principal["user_id"], profile or _active_profile())
-        except Exception as exc:  # noqa: BLE001
-            # A lookup that failed is not a lookup that said yes.
-            #
-            # This used to return MEMBER, on the reasoning that the caller had
-            # already passed a coarser permission check so a transient database
-            # error should not deny an authorised write. The effect was that any
-            # error in the role lookup -- a write-lock timeout, a checkpoint, a
-            # corrupt page -- promoted a viewer to a role that can write, at
-            # exactly the moment the store was under stress. A caller able to
-            # provoke lock contention could provoke the promotion.
-            #
-            # "Ask again in a moment" is the honest answer and the one the
-            # caller can act on. It is neither a denial nor a grant.
-            from fastapi import HTTPException
-
-            logger.warning(
-                "rbac: the role for this caller could not be read (%s); "
-                "answering 503 rather than assuming one", exc,
-            )
-            raise HTTPException(
-                status_code=503,
-                detail="the workspace's roles are temporarily unreadable; retry",
-            ) from exc
-    mapped = {
-        Role.ADMIN: ActorRole.ADMIN,
-        Role.MEMBER: ActorRole.MEMBER,
-        Role.VIEWER: ActorRole.VIEWER,
-    }.get(role)
-    return frozenset({mapped}) if mapped is not None else frozenset({ActorRole.ANONYMOUS})
-
-
-def principal_info(request: Request) -> dict:
-    """Rich identity for /whoami: principal + role + effective permissions on
-    the active profile. Never raises — used by the dashboard to render UI."""
-    rbac = get_rbac_engine(request.app.state)
-    principal = resolve_principal(request)
-    prof = _active_profile()
-    info = {
-        "kind": principal["kind"],
-        "user_id": principal["user_id"],
-        "username": principal["username"],
-        "display_name": principal.get("display_name", principal["username"]),
-        "profile": prof,
-        "rbac_active": bool(rbac is not None and rbac.user_count() > 0),
-        "require_login": bool(rbac is not None and rbac.require_login()),
-    }
-    if principal["kind"] == "owner":
-        # Owner has every permission (personal mode) unless login is required.
-        info["role"] = "owner"
-        info["permissions"] = [p.value for p in Permission]
-        return info
-    role = rbac.get_role(principal["user_id"], prof) if rbac is not None else None
-    info["role"] = role.value if role else None
-    info["permissions"] = (
-        [p.value for p in permissions_for_role(role)] if role else []
-    )
-    return info

package/src/superlocalmemory/server/recall_health.py

@@ -1,343 +0,0 @@
-"""Runtime recall-health monitor — keep the full recall path warm,
-detect a "warm-but-broken" embedder at runtime, and self-heal it (v3.6.8).
-
-Why this exists
----------------
-On a long-running daemon the cold full-fusion recall could exceed the MCP
-pool's 30s timeout, and ``session_init`` then silently fell back to FTS5/BM25
-("DEGRADED MODE") — observed 7×/2 days in production logs. Two root causes:
-
-1. **Page-cache eviction.** The ~100 MB ``association_edges`` / graph table is
-   evicted under memory pressure, so the first recall after idle re-reads it
-   from disk (15-24 s+), blowing the pool timeout.
-2. **Silent embedder death.** ``OllamaEmbedder.embed`` returns ``None`` on a
-   transient Ollama failure, while the boot-set ``_embedding_warm`` flag still
-   reports ``True``. With ``q_emb is None`` the engine skips the semantic,
-   hopfield and spreading_activation channels — recall silently degrades to
-   keyword-only (``semantic`` score ``0.0`` on every result).
-
-The boot warmup (``_warmup_recall``) runs ONCE and never again, so neither
-condition is repaired in-life.
-
-The fix — an industry-standard 3-tier monitor, validated against Ollama
-keep-alive, Chroma's active heartbeat, LangChain's circuit breaker and the
-Kubernetes liveness/readiness split:
-
-* **Tier 1 — RE-WARM.** Fire a real ``engine.recall`` every ``interval_s`` to
-  keep the graph page cache hot and nomic-embed resident.
-* **Tier 2 — READINESS PROBE.** Assert the semantic channel actually fired
-  (``max semantic > 0``). Rows returned with ``semantic == 0`` everywhere is the
-  warm-but-broken signature → the embedder is returning ``None``.
-* **Tier 3 — CIRCUIT-BREAKER SELF-HEAL.** Reset the embedder's cached
-  ``_available`` flag (the "available once, cached forever" bug), re-exercise
-  ``embed()``, track consecutive failures, and log **CRITICAL** so the
-  degradation is never silent.
-
-All logic is fail-soft: a crash in the monitor never takes down the daemon.
-"""
-from __future__ import annotations
-
-import logging
-import threading
-import time
-from contextlib import nullcontext
-from dataclasses import dataclass
-
-logger = logging.getLogger(__name__)
-
-# A stable probe query. Content-agnostic: we only assert that *some* result
-# carries a non-zero semantic score, which proves the embedder + semantic
-# channel are alive end-to-end.
-DEFAULT_PROBE = "memory recall health probe"
-# A distinct string for the heal embed() call so it is obvious in thread dumps
-# / Ollama logs and never collides with a real cached query embedding.
-HEAL_PROBE = "__recall_health_rewarm__"
-
-# Default cadence. 5 min keeps nomic-embed resident (Ollama default unload is
-# 5 min) and the page cache warm without meaningful load.
-DEFAULT_INTERVAL_S = 300
-
-
-@dataclass
-class RecallHealth:
-    """Mutable health state for the recall path."""
-
-    healthy: bool = True
-    consecutive_failures: int = 0
-    total_heals: int = 0
-    checks: int = 0
-    last_semantic_score: float = 0.0
-    last_error: str = ""
-    #: When the last tick finished, as a unix timestamp. A tick that finds
-    #: nothing wrong logs nothing, which is correct -- a monitor that narrates
-    #: every success is a monitor whose real warnings get skimmed past. But it
-    #: left no way to tell a monitor that is ticking quietly from a thread that
-    #: died or never started, and that ambiguity cost someone an hour of looking
-    #: for log lines that were never going to appear. So the fact of the tick is
-    #: recorded here and surfaced on /health, where it can be checked instead of
-    #: inferred.
-    last_tick_at: float = 0.0
-    #: Whether the embedder could produce a vector at the last tick.
-    embedder_alive: bool = True
-
-
-def _max_semantic(results) -> float:
-    """Largest semantic channel score across results (0.0 if none)."""
-    best = 0.0
-    for r in results:
-        cs = getattr(r, "channel_scores", None) or {}
-        try:
-            best = max(best, float(cs.get("semantic", 0.0) or 0.0))
-        except (TypeError, ValueError):
-            continue
-    return best
-
-
-def _get_embedder(engine):
-    """Locate the engine's embedder (full mode: engine._embedder; some paths
-    hang it off the retrieval engine)."""
-    emb = getattr(engine, "_embedder", None)
-    if emb is None:
-        re_eng = getattr(engine, "_retrieval_engine", None)
-        emb = getattr(re_eng, "_embedder", None) if re_eng is not None else None
-    return emb
-
-
-def _embedder_is_dead(engine) -> bool:
-    """Can the embedder produce a vector right now?
-
-    Asked directly, because it cannot be inferred from a recall. The monitor
-    used to decide the embedder was fine whenever the probe recall came back
-    with no results at all -- and a dead embedder is one of the reasons a recall
-    comes back with no results, so the one symptom that should have triggered a
-    heal was read as proof that none was needed.
-
-    That is not hypothetical. An idle-timeout kill leaves no worker; the next
-    probe finds nothing by meaning, finds nothing by keyword either because the
-    probe phrase appears in nobody's memories, and returns zero results. The
-    monitor then recorded "healthy", logged nothing, and never respawned the
-    worker -- so ``readiness.embedding`` stayed false and the daemon sat in
-    ``warming`` until someone restarted it by hand, with not one line in the log
-    to say why.
-
-    Fails safe in the opposite direction from before: an embedder this cannot
-    reach is reported dead, so the worst case is one unnecessary re-warm rather
-    than a silent outage.
-    """
-    emb = _get_embedder(engine)
-    if emb is None:
-        return False  # BM25-only by configuration; nothing to heal.
-    warm = getattr(emb, "is_warm", None)
-    if warm is not None and not warm:
-        return True
-    if getattr(emb, "_available", True) is False:
-        return True
-    return False
-
-
-def _heal_embedder(engine, *, log) -> bool:
-    """Tier 3: reset the cached availability flag and re-exercise the embedder.
-
-    ``OllamaEmbedder.is_available`` caches its first result forever, so once
-    Ollama blips the flag can stay stale. Clearing ``_available`` forces a
-    re-probe; ``embed()`` itself always re-attempts the HTTP call, so a single
-    successful embed proves recovery. Returns True iff the embedder produced a
-    vector.
-    """
-    emb = _get_embedder(engine)
-    if emb is None:
-        log.critical("recall-health: no embedder on engine — cannot self-heal")
-        return False
-    # Reset the "available once, cached forever" flag if present.
-    if hasattr(emb, "_available"):
-        try:
-            emb._available = None
-        except Exception:  # pragma: no cover - defensive
-            pass
-    try:
-        vec = emb.embed(HEAL_PROBE)
-    except Exception as exc:
-        log.warning("recall-health: heal embed() raised: %s", exc)
-        return False
-    return vec is not None and bool(vec)
-
-
-def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
-                    log=logger, runtime=None) -> RecallHealth:
-    """One monitor tick: re-warm (Tier 1), probe (Tier 2), self-heal (Tier 3).
-
-    Mutates and returns ``state``. Never raises — a timed-out / failing recall
-    marks the path unhealthy instead of propagating.
-
-    Uses ``operation_nowait()`` when a runtime is supplied so that a pending
-    profile switch is not blocked by the health-probe recall.  If a transition
-    is in progress the tick is skipped entirely — the next scheduled tick will
-    re-warm once the switch has committed.
-    """
-    state.checks += 1
-
-    # Tier 1: re-warm. A real full-fusion recall keeps the graph page cache hot
-    # and the embedder resident.
-    # Cooperative preemption: use operation_nowait() so a pending profile switch
-    # is not held hostage by a slow full-fusion recall (2–10s with fast=False).
-    try:
-        if runtime is not None:
-            lease = runtime.operation_nowait()
-        else:
-            lease = nullcontext()
-        with lease as _snap:
-            if runtime is not None and _snap is None:
-                # A profile transition is in progress — skip this tick so we
-                # do not hold the drain window.  Health state is unchanged;
-                # the next tick fires after the switch commits.
-                log.debug(
-                    "recall-health: tick skipped — profile transition in progress"
-                )
-                return state
-            # fast=True: a health probe must release its operation lease well
-            # within the 5s profile-switch drain window (fast=False is 2-10s and
-            # would make every profile switch time out while a tick is in flight).
-            from superlocalmemory.core.recall_gate import background_work
-            with background_work():
-                resp = engine.recall(probe, limit=3, fast=True)
-    except Exception as exc:
-        state.healthy = False
-        state.consecutive_failures += 1
-        state.last_error = f"recall raised: {exc}"
-        log.critical(
-            "recall-health: re-warm recall FAILED (%s) — recall path unhealthy",
-            exc,
-        )
-        return state
-
-    results = list(getattr(resp, "results", []) or [])
-    sem = _max_semantic(results)
-    state.last_semantic_score = sem
-    state.last_tick_at = time.time()
-
-    # Tier 2: readiness. Two independent signatures, and the second one is why
-    # this monitor exists.
-    #
-    #   * rows present but semantic never fired  -> warm-but-broken
-    #   * the embedder cannot produce a vector   -> dead, whatever the recall said
-    #
-    # The second used to be missing, and its absence was load-bearing: zero
-    # results was treated as "not this signature", so the case where the embedder
-    # is dead AND the probe matches nothing by keyword -- which is the normal
-    # shape of an idle-timeout kill -- came out as healthy, silently.
-    dead = _embedder_is_dead(engine)
-    state.embedder_alive = not dead
-    broken = dead or (bool(results) and sem <= 0.0)
-    if dead:
-        log.critical(
-            "recall-health: embedder cannot produce a vector (%d probe results) "
-            "— attempting self-heal",
-            len(results),
-        )
-    if not broken:
-        if not state.healthy:
-            log.warning(
-                "recall-health: RECOVERED (semantic=%.3f, %d results)",
-                sem, len(results),
-            )
-        state.healthy = True
-        state.consecutive_failures = 0
-        state.last_error = ""
-        return state
-
-    # Tier 3: self-heal. The dead-embedder case already said so above; saying
-    # "semantic channel DEAD (max semantic=0.0)" as well would be a second,
-    # differently-worded CRITICAL about the same tick, and one of the two would
-    # be describing a symptom the reader does not have.
-    if not dead:
-        log.critical(
-            "recall-health: semantic channel DEAD (%d results, max semantic=0.0) "
-            "— embedder returning None; attempting self-heal",
-            len(results),
-        )
-    if _heal_embedder(engine, log=log):
-        state.total_heals += 1
-        state.healthy = True
-        state.consecutive_failures = 0
-        state.last_error = ""
-        log.warning("recall-health: embedder self-heal SUCCEEDED (re-warmed)")
-    else:
-        state.healthy = False
-        state.consecutive_failures += 1
-        state.last_error = "semantic channel dead; embedder heal failed"
-        log.critical(
-            "recall-health: self-heal FAILED — recall DEGRADED to keyword-only "
-            "(consecutive_failures=%d)", state.consecutive_failures,
-        )
-    return state
-
-
-def health_monitor_loop(engine, *, interval_s: int, stop_event: threading.Event,
-                        state: RecallHealth, probe: str = DEFAULT_PROBE,
-                        log=logger, runtime=None) -> None:
-    """Background loop. Sleeps ``interval_s`` between ticks; exits promptly when
-    ``stop_event`` is set. An initial short delay avoids racing boot warmup."""
-    # Initial delay (bounded) so we don't pile onto the boot warmup threads.
-    if stop_event.wait(min(interval_s, 60)):
-        return
-    while not stop_event.is_set():
-        try:
-            run_health_tick(
-                engine, state, probe=probe, log=log, runtime=runtime,
-            )
-        except Exception as exc:  # pragma: no cover - belt & suspenders
-            log.warning("recall-health: tick crashed (non-fatal): %s", exc)
-        if stop_event.wait(interval_s):
-            break
-
-
-# Module-level state so /health can surface the latest verdict without holding
-# a reference to the thread.
-_GLOBAL_STATE = RecallHealth()
-
-
-def start_recall_health_monitor(engine, *, interval_s: int = DEFAULT_INTERVAL_S,
-                                probe: str = DEFAULT_PROBE, log=None,
-                                runtime=None):
-    """Start the monitor as a daemon thread. Returns ``(thread, stop_event,
-    state)``. The state is the shared module-level state read by
-    :func:`get_recall_health`."""
-    log = log or logger
-    state = _GLOBAL_STATE
-    stop = threading.Event()
-    t = threading.Thread(
-        target=health_monitor_loop,
-        kwargs=dict(
-            engine=engine, interval_s=interval_s, stop_event=stop,
-            state=state, probe=probe, log=log, runtime=runtime,
-        ),
-        daemon=True,
-        name="recall-health",
-    )
-    t.start()
-    return t, stop, state
-
-
-def get_recall_health() -> dict:
-    """Snapshot for /health surfacing (visibility — never silent degradation)."""
-    s = _GLOBAL_STATE
-    now = time.time()
-    return {
-        "recall_healthy": s.healthy,
-        "consecutive_failures": s.consecutive_failures,
-        "total_heals": s.total_heals,
-        "checks": s.checks,
-        "last_semantic_score": round(s.last_semantic_score, 4),
-        "last_error": s.last_error,
-        # Proof of life. A tick that finds nothing wrong logs nothing, so there
-        # was no way to tell this monitor apart from a thread that never started
-        # -- someone spent an hour reading logs for lines that were never going
-        # to be written. These two answer that without needing the log at all:
-        # if seconds_since_last_tick keeps climbing past the interval, the thread
-        # is gone.
-        "last_tick_at": round(s.last_tick_at, 3) if s.last_tick_at else None,
-        "seconds_since_last_tick": (
-            round(now - s.last_tick_at, 1) if s.last_tick_at else None
-        ),
-        "embedder_alive": s.embedder_alive,
-    }