superlocalmemory 4.0.10 → 4.1.2

package/src/superlocalmemory/server/bandit_loops.py

@@ -68,19 +68,35 @@ async def _reward_proxy_loop(
     interval_sec: float,
 ) -> None:
     """Run the proxy settler on a steady interval. Never raises."""
+    from superlocalmemory.learning.reward_from_outcomes import (
+        settle_from_outcomes,
+    )
     from superlocalmemory.learning.reward_proxy import settle_stale_plays
 
     while True:
         try:
             await asyncio.sleep(interval_sec)
+            # Reported outcomes FIRST. The proxy defaults a play once its
+            # window closes; if it ran first it would claim every play and a
+            # real outcome arriving later would find nothing to settle.
+            real = await asyncio.to_thread(
+                settle_from_outcomes,
+                profile_id, learning_db, memory_db,
+            )
             # The settler is synchronous + fast; run in a thread to avoid
             # blocking the event loop on unusual DB lock stalls.
             n = await asyncio.to_thread(
                 settle_stale_plays,
                 profile_id, learning_db, memory_db,
             )
+            # INFO, not DEBUG, when a real outcome moved an arm: this is the
+            # only line that distinguishes a learning loop that is running
+            # from one that merely starts. Its absence for a month is what
+            # made this defect invisible.
+            if real:
+                logger.info("bandit settled %d play(s) from outcomes", real)
             if n:
-                logger.debug("bandit.reward_proxy settled=%d", n)
+                logger.debug("bandit.reward_proxy settled=%d (default)", n)
         except asyncio.CancelledError:  # pragma: no cover — lifecycle
             raise
         except Exception as exc:  # pragma: no cover — defensive

package/src/superlocalmemory/server/rbac_enforce.py

@@ -27,6 +27,9 @@ from typing import Any
 from fastapi import HTTPException, Request
 
 from superlocalmemory.access.rbac import Permission, Role, permissions_for_role
+import logging
+
+logger = logging.getLogger(__name__)
 
 _SESSION_HEADER = "X-SLM-User-Session"
 _SESSION_COOKIE = "slm_session"
@@ -132,12 +135,29 @@ def resolve_actor_roles(request: Request, *, profile: str | None = None):
     if rbac is not None:
         try:
             role = rbac.get_role(principal["user_id"], profile or _active_profile())
-        except Exception:
-            # The caller already passed require_permission for this operation, so
-            # a transient role lookup must not surface as a 500. Fall back to the
-            # least-privileged write-capable role rather than deny an authorized
-            # write.
-            return frozenset({ActorRole.MEMBER})
+        except Exception as exc:  # noqa: BLE001
+            # A lookup that failed is not a lookup that said yes.
+            #
+            # This used to return MEMBER, on the reasoning that the caller had
+            # already passed a coarser permission check so a transient database
+            # error should not deny an authorised write. The effect was that any
+            # error in the role lookup -- a write-lock timeout, a checkpoint, a
+            # corrupt page -- promoted a viewer to a role that can write, at
+            # exactly the moment the store was under stress. A caller able to
+            # provoke lock contention could provoke the promotion.
+            #
+            # "Ask again in a moment" is the honest answer and the one the
+            # caller can act on. It is neither a denial nor a grant.
+            from fastapi import HTTPException
+
+            logger.warning(
+                "rbac: the role for this caller could not be read (%s); "
+                "answering 503 rather than assuming one", exc,
+            )
+            raise HTTPException(
+                status_code=503,
+                detail="the workspace's roles are temporarily unreadable; retry",
+            ) from exc
     mapped = {
         Role.ADMIN: ActorRole.ADMIN,
         Role.MEMBER: ActorRole.MEMBER,

package/src/superlocalmemory/server/recall_health.py

@@ -39,6 +39,7 @@ from __future__ import annotations
 
 import logging
 import threading
+import time
 from contextlib import nullcontext
 from dataclasses import dataclass
 
@@ -67,6 +68,17 @@ class RecallHealth:
     checks: int = 0
     last_semantic_score: float = 0.0
     last_error: str = ""
+    #: When the last tick finished, as a unix timestamp. A tick that finds
+    #: nothing wrong logs nothing, which is correct -- a monitor that narrates
+    #: every success is a monitor whose real warnings get skimmed past. But it
+    #: left no way to tell a monitor that is ticking quietly from a thread that
+    #: died or never started, and that ambiguity cost someone an hour of looking
+    #: for log lines that were never going to appear. So the fact of the tick is
+    #: recorded here and surfaced on /health, where it can be checked instead of
+    #: inferred.
+    last_tick_at: float = 0.0
+    #: Whether the embedder could produce a vector at the last tick.
+    embedder_alive: bool = True
 
 
 def _max_semantic(results) -> float:
@@ -91,6 +103,38 @@ def _get_embedder(engine):
     return emb
 
 
+def _embedder_is_dead(engine) -> bool:
+    """Can the embedder produce a vector right now?
+
+    Asked directly, because it cannot be inferred from a recall. The monitor
+    used to decide the embedder was fine whenever the probe recall came back
+    with no results at all -- and a dead embedder is one of the reasons a recall
+    comes back with no results, so the one symptom that should have triggered a
+    heal was read as proof that none was needed.
+
+    That is not hypothetical. An idle-timeout kill leaves no worker; the next
+    probe finds nothing by meaning, finds nothing by keyword either because the
+    probe phrase appears in nobody's memories, and returns zero results. The
+    monitor then recorded "healthy", logged nothing, and never respawned the
+    worker -- so ``readiness.embedding`` stayed false and the daemon sat in
+    ``warming`` until someone restarted it by hand, with not one line in the log
+    to say why.
+
+    Fails safe in the opposite direction from before: an embedder this cannot
+    reach is reported dead, so the worst case is one unnecessary re-warm rather
+    than a silent outage.
+    """
+    emb = _get_embedder(engine)
+    if emb is None:
+        return False  # BM25-only by configuration; nothing to heal.
+    warm = getattr(emb, "is_warm", None)
+    if warm is not None and not warm:
+        return True
+    if getattr(emb, "_available", True) is False:
+        return True
+    return False
+
+
 def _heal_embedder(engine, *, log) -> bool:
     """Tier 3: reset the cached availability flag and re-exercise the embedder.
 
@@ -169,10 +213,27 @@ def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
     results = list(getattr(resp, "results", []) or [])
     sem = _max_semantic(results)
     state.last_semantic_score = sem
-
-    # Tier 2: readiness. Rows present but semantic never fired == warm-but-broken.
-    # Zero results is NOT this signature (could be an empty/filtered corpus).
-    broken = bool(results) and sem <= 0.0
+    state.last_tick_at = time.time()
+
+    # Tier 2: readiness. Two independent signatures, and the second one is why
+    # this monitor exists.
+    #
+    #   * rows present but semantic never fired  -> warm-but-broken
+    #   * the embedder cannot produce a vector   -> dead, whatever the recall said
+    #
+    # The second used to be missing, and its absence was load-bearing: zero
+    # results was treated as "not this signature", so the case where the embedder
+    # is dead AND the probe matches nothing by keyword -- which is the normal
+    # shape of an idle-timeout kill -- came out as healthy, silently.
+    dead = _embedder_is_dead(engine)
+    state.embedder_alive = not dead
+    broken = dead or (bool(results) and sem <= 0.0)
+    if dead:
+        log.critical(
+            "recall-health: embedder cannot produce a vector (%d probe results) "
+            "— attempting self-heal",
+            len(results),
+        )
     if not broken:
         if not state.healthy:
             log.warning(
@@ -184,12 +245,16 @@ def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
         state.last_error = ""
         return state
 
-    # Tier 3: self-heal.
-    log.critical(
-        "recall-health: semantic channel DEAD (%d results, max semantic=0.0) "
-        "— embedder returning None; attempting self-heal",
-        len(results),
-    )
+    # Tier 3: self-heal. The dead-embedder case already said so above; saying
+    # "semantic channel DEAD (max semantic=0.0)" as well would be a second,
+    # differently-worded CRITICAL about the same tick, and one of the two would
+    # be describing a symptom the reader does not have.
+    if not dead:
+        log.critical(
+            "recall-health: semantic channel DEAD (%d results, max semantic=0.0) "
+            "— embedder returning None; attempting self-heal",
+            len(results),
+        )
     if _heal_embedder(engine, log=log):
         state.total_heals += 1
         state.healthy = True
@@ -256,6 +321,7 @@ def start_recall_health_monitor(engine, *, interval_s: int = DEFAULT_INTERVAL_S,
 def get_recall_health() -> dict:
     """Snapshot for /health surfacing (visibility — never silent degradation)."""
     s = _GLOBAL_STATE
+    now = time.time()
     return {
         "recall_healthy": s.healthy,
         "consecutive_failures": s.consecutive_failures,
@@ -263,4 +329,15 @@ def get_recall_health() -> dict:
         "checks": s.checks,
         "last_semantic_score": round(s.last_semantic_score, 4),
         "last_error": s.last_error,
+        # Proof of life. A tick that finds nothing wrong logs nothing, so there
+        # was no way to tell this monitor apart from a thread that never started
+        # -- someone spent an hour reading logs for lines that were never going
+        # to be written. These two answer that without needing the log at all:
+        # if seconds_since_last_tick keeps climbing past the interval, the thread
+        # is gone.
+        "last_tick_at": round(s.last_tick_at, 3) if s.last_tick_at else None,
+        "seconds_since_last_tick": (
+            round(now - s.last_tick_at, 1) if s.last_tick_at else None
+        ),
+        "embedder_alive": s.embedder_alive,
     }

package/src/superlocalmemory/server/recall_serializer.py

@@ -291,6 +291,10 @@ def recall_response_metadata(response: Any) -> dict:
         "score_contract_version": getattr(response, "score_contract_version", "2"),
         "calibration_status": getattr(response, "calibration_status", "uncalibrated"),
         "calibration_id": getattr(response, "calibration_id", None),
+        # The name of this answer. A caller that reports back how the answer
+        # went can quote it, and the report then joins to this exact recall
+        # instead of being matched by overlapping memory ids.
+        "query_id": getattr(response, "query_id", "") or "",
         "answer_confidence": getattr(response, "answer_confidence", None),
         "abstained": bool(getattr(response, "abstained", False)),
         "abstention_reason": getattr(response, "abstention_reason", None),
@@ -308,4 +312,9 @@ def recall_response_metadata(response: Any) -> dict:
         "incomplete_channels": list(
             getattr(response, "incomplete_channels", ()) or ()
         ),
+        # What became of every channel. Travels with the answer for the same
+        # reason as the field above: a caller comparing two runs, or an
+        # operator looking at a thin result set, otherwise cannot tell a store
+        # with nothing to say from a retrieval path that is partly down.
+        "channel_status": dict(getattr(response, "channel_status", {}) or {}),
     }

package/src/superlocalmemory/server/routes/behavioral.py

@@ -29,6 +29,66 @@ _MAX_OUTCOME_FACT_IDS = 100
 _MAX_FACT_ID_LENGTH = 200
 
 
+def _outcome_insert(
+    conn,
+    *,
+    outcome_id: str,
+    profile: str,
+    fact_ids_json: str,
+    outcome: str,
+    context_json: str,
+    now_iso: str,
+    reward: float,
+    recall_query_id: str,
+) -> tuple[str, tuple]:
+    """Build the ``action_outcomes`` INSERT from the columns that exist.
+
+    ``reward``, ``settled``, ``settled_at`` and ``recall_query_id`` all arrive
+    with M006, which is a DEFERRED migration — it runs after engine init, not
+    during ``apply_all``. This handler has always named them unconditionally, so
+    on a store where M006 has not run it raised ``no such column`` and returned
+    ``success: false`` for a write that could perfectly well have been recorded
+    without them.
+
+    ``recall_query_id`` is the one that matters for learning: it is what lets a
+    bandit play be settled from this report. The column has existed since M006
+    and was never written — 0 of 162 rows on a live store carry one — so
+    every play fell through to the neutral 120-second default and 165 arms sat
+    at alpha == beta. Recording it is the join key that closes the loop.
+
+    Degrading per column rather than per statement means an unmigrated store
+    still records the outcome; it just cannot attribute it to a play, which is
+    the honest reduction in capability.
+    """
+    columns = ["outcome_id", "profile_id", "query", "fact_ids_json",
+               "outcome", "context_json", "timestamp"]
+    values: list = [outcome_id, profile, "", fact_ids_json, outcome,
+                    context_json, now_iso]
+    try:
+        present = {
+            row[1] for row in conn.execute("PRAGMA table_info(action_outcomes)")
+        }
+    except Exception:  # pragma: no cover — defensive
+        present = set()
+
+    for name, value in (
+        ("reward", reward),
+        ("settled", 1),
+        ("settled_at", now_iso),
+        ("recall_query_id", recall_query_id),
+    ):
+        if name in present:
+            columns.append(name)
+            values.append(value)
+
+    placeholders = ", ".join("?" * len(columns))
+    sql = (
+        f"INSERT INTO action_outcomes ({', '.join(columns)}) "
+        f"VALUES ({placeholders})"
+    )
+    return sql, tuple(values)
+
+
 class ReportOutcomeRequest(BaseModel):
     """Bounded explicit outcome payload accepted from dashboard clients."""
 
@@ -41,6 +101,11 @@ class ReportOutcomeRequest(BaseModel):
     outcome: Literal["success", "failure", "partial"]
     action_type: StrictStr = Field(default="other", max_length=80)
     context: StrictStr = Field(default="", max_length=1000)
+    #: Which recall this outcome judges, if the caller knows. Optional because
+    #: recall does not currently return its query_id to callers, so nothing can
+    #: supply one yet; when it is absent the settler falls back to overlap
+    #: between ``memory_ids`` and the memories a play recorded showing.
+    recall_query_id: StrictStr = Field(default="", max_length=64)
 
     @field_validator("memory_ids")
     @classmethod
@@ -394,16 +459,16 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
                 fact_ids=memory_ids,
             )
             conn.execute(
-                "INSERT INTO action_outcomes "
-                "(outcome_id, profile_id, query, fact_ids_json, outcome, "
-                " context_json, timestamp, reward, settled, settled_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
-                (
-                    outcome_id, profile, "",
-                    json.dumps(memory_ids),
-                    outcome,
-                    json.dumps(context_dict),
-                    now_iso, reward, now_iso,
+                *_outcome_insert(
+                    conn,
+                    outcome_id=outcome_id,
+                    profile=profile,
+                    fact_ids_json=json.dumps(memory_ids),
+                    outcome=outcome,
+                    context_json=json.dumps(context_dict),
+                    now_iso=now_iso,
+                    reward=reward,
+                    recall_query_id=data.recall_query_id,
                 ),
             )
 

package/src/superlocalmemory/server/routes/compliance.py

@@ -40,9 +40,33 @@ except ImportError:
     logger.info("V3 compliance engine not available")
 
 
+def _require_manage(request) -> None:
+    """Require MANAGE on the active workspace, where the workspace has roles.
+
+    A store with no roles configured has nobody to check against, and refusing
+    there would break every personal install; the loopback boundary is the
+    control in that case.
+    """
+    try:
+        from superlocalmemory.access.rbac import Permission
+        from superlocalmemory.server.rbac_enforce import require_permission
+
+        require_permission(request, Permission.MANAGE, profile=get_active_profile())
+    except ImportError:  # pragma: no cover -- roles are always present in-tree
+        return
+
+
 @router.get("/api/compliance/status")
-async def compliance_status():
+async def compliance_status(request: Request):
     """Get compliance engine status for active profile."""
+    # This returns which operations ran and what categories of memory exist.
+    # That is operational metadata about someone's store; it had no gate at all.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="compliance-read",
+    )
     if not COMPLIANCE_AVAILABLE:
         return {"available": False, "message": "Compliance engine not available"}
 
@@ -132,8 +156,43 @@ async def query_audit_trail(
         return {"available": False, "error": "Internal server error"}
 
 
+def _erasure_succeeded(result: dict) -> bool:
+    """One place that decides, so the surfaces cannot disagree.
+
+    The erasure reports two things, because they are two questions: whether the
+    data is gone, and whether that can be shown afterwards. This route used to
+    inline its own list of failure markers and the CLI used a different one, so
+    the API could say failure while the command line printed COMPLETE for the
+    same erasure.
+
+    Anything that leaves either question unanswered is a failure here. A caller
+    that wants the finer distinction reads ``erasure_complete`` and
+    ``erasure_provable`` from the body, which are both in it.
+    """
+    markers = (
+        "vector_store_failures",
+        "audit_completion_failed",
+        "audit_request_failed",
+        "receipt_persist_failed",
+        "table_delete_failures",
+        "code_graph_failed",
+        "fact_expansion_fts_failed",
+        "working_sets_failed",
+        "residue_recount_failed",
+        "backup_scan_failed",
+    )
+    if any(result.get(marker) for marker in markers):
+        return False
+    # Present on the profile path; absent on the entity path, where its absence
+    # must not be read as a failure.
+    for verdict in ("erasure_complete", "erasure_provable"):
+        if verdict in result and not result.get(verdict):
+            return False
+    return True
+
+
 @router.post("/api/compliance/retention-policy")
-async def create_retention_policy(data: dict):
+async def create_retention_policy(request: Request, data: dict):
     """Create a compliance retention policy.
 
     Body: {
@@ -144,6 +203,17 @@ async def create_retention_policy(data: dict):
         applies_to: dict (optional)
     }
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
 
@@ -183,8 +253,19 @@ async def create_retention_policy(data: dict):
 
 
 @router.delete("/api/compliance/retention-policy")
-async def delete_retention_policy(name: str = Query(...)):
+async def delete_retention_policy(request: Request, name: str = Query(...)):
     """Delete a retention policy by name for the active profile."""
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -202,12 +283,23 @@ async def delete_retention_policy(name: str = Query(...)):
 
 
 @router.post("/api/compliance/retention/enforce")
-async def enforce_retention():
+async def enforce_retention(request: Request):
     """Run all retention policies for the active profile now.
 
     Moves expired facts to their rule's terminal lifecycle zone (archive/
     tombstone) or counts them (notify). Soft-state only — never a raw delete.
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -302,13 +394,7 @@ async def gdpr_erase(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_profile(profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {"success": success, "active_profile": profile, **result}
     except Exception:
         logger.exception("gdpr_erase error")
@@ -361,13 +447,7 @@ async def gdpr_erase_entity(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_entity(entity_name, profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {
             "success": success, "active_profile": profile,
             "entity_name": entity_name, **result,