superlocalmemory 4.0.10 → 4.1.0

package/src/superlocalmemory/server/bandit_loops.py

@@ -68,19 +68,35 @@ async def _reward_proxy_loop(
     interval_sec: float,
 ) -> None:
     """Run the proxy settler on a steady interval. Never raises."""
+    from superlocalmemory.learning.reward_from_outcomes import (
+        settle_from_outcomes,
+    )
     from superlocalmemory.learning.reward_proxy import settle_stale_plays
 
     while True:
         try:
             await asyncio.sleep(interval_sec)
+            # Reported outcomes FIRST. The proxy defaults a play once its
+            # window closes; if it ran first it would claim every play and a
+            # real outcome arriving later would find nothing to settle.
+            real = await asyncio.to_thread(
+                settle_from_outcomes,
+                profile_id, learning_db, memory_db,
+            )
             # The settler is synchronous + fast; run in a thread to avoid
             # blocking the event loop on unusual DB lock stalls.
             n = await asyncio.to_thread(
                 settle_stale_plays,
                 profile_id, learning_db, memory_db,
             )
+            # INFO, not DEBUG, when a real outcome moved an arm: this is the
+            # only line that distinguishes a learning loop that is running
+            # from one that merely starts. Its absence for a month is what
+            # made this defect invisible.
+            if real:
+                logger.info("bandit settled %d play(s) from outcomes", real)
             if n:
-                logger.debug("bandit.reward_proxy settled=%d", n)
+                logger.debug("bandit.reward_proxy settled=%d (default)", n)
         except asyncio.CancelledError:  # pragma: no cover — lifecycle
             raise
         except Exception as exc:  # pragma: no cover — defensive

package/src/superlocalmemory/server/rbac_enforce.py

@@ -27,6 +27,9 @@ from typing import Any
 from fastapi import HTTPException, Request
 
 from superlocalmemory.access.rbac import Permission, Role, permissions_for_role
+import logging
+
+logger = logging.getLogger(__name__)
 
 _SESSION_HEADER = "X-SLM-User-Session"
 _SESSION_COOKIE = "slm_session"
@@ -132,12 +135,29 @@ def resolve_actor_roles(request: Request, *, profile: str | None = None):
     if rbac is not None:
         try:
             role = rbac.get_role(principal["user_id"], profile or _active_profile())
-        except Exception:
-            # The caller already passed require_permission for this operation, so
-            # a transient role lookup must not surface as a 500. Fall back to the
-            # least-privileged write-capable role rather than deny an authorized
-            # write.
-            return frozenset({ActorRole.MEMBER})
+        except Exception as exc:  # noqa: BLE001
+            # A lookup that failed is not a lookup that said yes.
+            #
+            # This used to return MEMBER, on the reasoning that the caller had
+            # already passed a coarser permission check so a transient database
+            # error should not deny an authorised write. The effect was that any
+            # error in the role lookup -- a write-lock timeout, a checkpoint, a
+            # corrupt page -- promoted a viewer to a role that can write, at
+            # exactly the moment the store was under stress. A caller able to
+            # provoke lock contention could provoke the promotion.
+            #
+            # "Ask again in a moment" is the honest answer and the one the
+            # caller can act on. It is neither a denial nor a grant.
+            from fastapi import HTTPException
+
+            logger.warning(
+                "rbac: the role for this caller could not be read (%s); "
+                "answering 503 rather than assuming one", exc,
+            )
+            raise HTTPException(
+                status_code=503,
+                detail="the workspace's roles are temporarily unreadable; retry",
+            ) from exc
     mapped = {
         Role.ADMIN: ActorRole.ADMIN,
         Role.MEMBER: ActorRole.MEMBER,

package/src/superlocalmemory/server/recall_serializer.py

@@ -291,6 +291,10 @@ def recall_response_metadata(response: Any) -> dict:
         "score_contract_version": getattr(response, "score_contract_version", "2"),
         "calibration_status": getattr(response, "calibration_status", "uncalibrated"),
         "calibration_id": getattr(response, "calibration_id", None),
+        # The name of this answer. A caller that reports back how the answer
+        # went can quote it, and the report then joins to this exact recall
+        # instead of being matched by overlapping memory ids.
+        "query_id": getattr(response, "query_id", "") or "",
         "answer_confidence": getattr(response, "answer_confidence", None),
         "abstained": bool(getattr(response, "abstained", False)),
         "abstention_reason": getattr(response, "abstention_reason", None),
@@ -308,4 +312,9 @@ def recall_response_metadata(response: Any) -> dict:
         "incomplete_channels": list(
             getattr(response, "incomplete_channels", ()) or ()
         ),
+        # What became of every channel. Travels with the answer for the same
+        # reason as the field above: a caller comparing two runs, or an
+        # operator looking at a thin result set, otherwise cannot tell a store
+        # with nothing to say from a retrieval path that is partly down.
+        "channel_status": dict(getattr(response, "channel_status", {}) or {}),
     }

package/src/superlocalmemory/server/routes/behavioral.py

@@ -29,6 +29,66 @@ _MAX_OUTCOME_FACT_IDS = 100
 _MAX_FACT_ID_LENGTH = 200
 
 
+def _outcome_insert(
+    conn,
+    *,
+    outcome_id: str,
+    profile: str,
+    fact_ids_json: str,
+    outcome: str,
+    context_json: str,
+    now_iso: str,
+    reward: float,
+    recall_query_id: str,
+) -> tuple[str, tuple]:
+    """Build the ``action_outcomes`` INSERT from the columns that exist.
+
+    ``reward``, ``settled``, ``settled_at`` and ``recall_query_id`` all arrive
+    with M006, which is a DEFERRED migration — it runs after engine init, not
+    during ``apply_all``. This handler has always named them unconditionally, so
+    on a store where M006 has not run it raised ``no such column`` and returned
+    ``success: false`` for a write that could perfectly well have been recorded
+    without them.
+
+    ``recall_query_id`` is the one that matters for learning: it is what lets a
+    bandit play be settled from this report. The column has existed since M006
+    and was never written — 0 of 162 rows on a live store carry one — so
+    every play fell through to the neutral 120-second default and 165 arms sat
+    at alpha == beta. Recording it is the join key that closes the loop.
+
+    Degrading per column rather than per statement means an unmigrated store
+    still records the outcome; it just cannot attribute it to a play, which is
+    the honest reduction in capability.
+    """
+    columns = ["outcome_id", "profile_id", "query", "fact_ids_json",
+               "outcome", "context_json", "timestamp"]
+    values: list = [outcome_id, profile, "", fact_ids_json, outcome,
+                    context_json, now_iso]
+    try:
+        present = {
+            row[1] for row in conn.execute("PRAGMA table_info(action_outcomes)")
+        }
+    except Exception:  # pragma: no cover — defensive
+        present = set()
+
+    for name, value in (
+        ("reward", reward),
+        ("settled", 1),
+        ("settled_at", now_iso),
+        ("recall_query_id", recall_query_id),
+    ):
+        if name in present:
+            columns.append(name)
+            values.append(value)
+
+    placeholders = ", ".join("?" * len(columns))
+    sql = (
+        f"INSERT INTO action_outcomes ({', '.join(columns)}) "
+        f"VALUES ({placeholders})"
+    )
+    return sql, tuple(values)
+
+
 class ReportOutcomeRequest(BaseModel):
     """Bounded explicit outcome payload accepted from dashboard clients."""
 
@@ -41,6 +101,11 @@ class ReportOutcomeRequest(BaseModel):
     outcome: Literal["success", "failure", "partial"]
     action_type: StrictStr = Field(default="other", max_length=80)
     context: StrictStr = Field(default="", max_length=1000)
+    #: Which recall this outcome judges, if the caller knows. Optional because
+    #: recall does not currently return its query_id to callers, so nothing can
+    #: supply one yet; when it is absent the settler falls back to overlap
+    #: between ``memory_ids`` and the memories a play recorded showing.
+    recall_query_id: StrictStr = Field(default="", max_length=64)
 
     @field_validator("memory_ids")
     @classmethod
@@ -394,16 +459,16 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
                 fact_ids=memory_ids,
             )
             conn.execute(
-                "INSERT INTO action_outcomes "
-                "(outcome_id, profile_id, query, fact_ids_json, outcome, "
-                " context_json, timestamp, reward, settled, settled_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
-                (
-                    outcome_id, profile, "",
-                    json.dumps(memory_ids),
-                    outcome,
-                    json.dumps(context_dict),
-                    now_iso, reward, now_iso,
+                *_outcome_insert(
+                    conn,
+                    outcome_id=outcome_id,
+                    profile=profile,
+                    fact_ids_json=json.dumps(memory_ids),
+                    outcome=outcome,
+                    context_json=json.dumps(context_dict),
+                    now_iso=now_iso,
+                    reward=reward,
+                    recall_query_id=data.recall_query_id,
                 ),
             )
 

package/src/superlocalmemory/server/routes/compliance.py

@@ -40,9 +40,33 @@ except ImportError:
     logger.info("V3 compliance engine not available")
 
 
+def _require_manage(request) -> None:
+    """Require MANAGE on the active workspace, where the workspace has roles.
+
+    A store with no roles configured has nobody to check against, and refusing
+    there would break every personal install; the loopback boundary is the
+    control in that case.
+    """
+    try:
+        from superlocalmemory.access.rbac import Permission
+        from superlocalmemory.server.rbac_enforce import require_permission
+
+        require_permission(request, Permission.MANAGE, profile=get_active_profile())
+    except ImportError:  # pragma: no cover -- roles are always present in-tree
+        return
+
+
 @router.get("/api/compliance/status")
-async def compliance_status():
+async def compliance_status(request: Request):
     """Get compliance engine status for active profile."""
+    # This returns which operations ran and what categories of memory exist.
+    # That is operational metadata about someone's store; it had no gate at all.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="compliance-read",
+    )
     if not COMPLIANCE_AVAILABLE:
         return {"available": False, "message": "Compliance engine not available"}
 
@@ -132,8 +156,43 @@ async def query_audit_trail(
         return {"available": False, "error": "Internal server error"}
 
 
+def _erasure_succeeded(result: dict) -> bool:
+    """One place that decides, so the surfaces cannot disagree.
+
+    The erasure reports two things, because they are two questions: whether the
+    data is gone, and whether that can be shown afterwards. This route used to
+    inline its own list of failure markers and the CLI used a different one, so
+    the API could say failure while the command line printed COMPLETE for the
+    same erasure.
+
+    Anything that leaves either question unanswered is a failure here. A caller
+    that wants the finer distinction reads ``erasure_complete`` and
+    ``erasure_provable`` from the body, which are both in it.
+    """
+    markers = (
+        "vector_store_failures",
+        "audit_completion_failed",
+        "audit_request_failed",
+        "receipt_persist_failed",
+        "table_delete_failures",
+        "code_graph_failed",
+        "fact_expansion_fts_failed",
+        "working_sets_failed",
+        "residue_recount_failed",
+        "backup_scan_failed",
+    )
+    if any(result.get(marker) for marker in markers):
+        return False
+    # Present on the profile path; absent on the entity path, where its absence
+    # must not be read as a failure.
+    for verdict in ("erasure_complete", "erasure_provable"):
+        if verdict in result and not result.get(verdict):
+            return False
+    return True
+
+
 @router.post("/api/compliance/retention-policy")
-async def create_retention_policy(data: dict):
+async def create_retention_policy(request: Request, data: dict):
     """Create a compliance retention policy.
 
     Body: {
@@ -144,6 +203,17 @@ async def create_retention_policy(data: dict):
         applies_to: dict (optional)
     }
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
 
@@ -183,8 +253,19 @@ async def create_retention_policy(data: dict):
 
 
 @router.delete("/api/compliance/retention-policy")
-async def delete_retention_policy(name: str = Query(...)):
+async def delete_retention_policy(request: Request, name: str = Query(...)):
     """Delete a retention policy by name for the active profile."""
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -202,12 +283,23 @@ async def delete_retention_policy(name: str = Query(...)):
 
 
 @router.post("/api/compliance/retention/enforce")
-async def enforce_retention():
+async def enforce_retention(request: Request):
     """Run all retention policies for the active profile now.
 
     Moves expired facts to their rule's terminal lifecycle zone (archive/
     tombstone) or counts them (notify). Soft-state only — never a raw delete.
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -302,13 +394,7 @@ async def gdpr_erase(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_profile(profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {"success": success, "active_profile": profile, **result}
     except Exception:
         logger.exception("gdpr_erase error")
@@ -361,13 +447,7 @@ async def gdpr_erase_entity(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_entity(entity_name, profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {
             "success": success, "active_profile": profile,
             "entity_name": entity_name, **result,

package/src/superlocalmemory/server/routes/config_api.py

@@ -136,7 +136,11 @@ class ForgettingConfigUpdate(BaseModel):
     forget_threshold: Optional[float] = Field(None, ge=0.0, le=1.0)
     learning_rate: Optional[float] = Field(None, gt=0.0)
     forgetting_drift_scale: Optional[float] = Field(None, gt=0.0)
-    trust_kappa: Optional[float] = Field(None, gt=0.0)
+    # ge, not gt: zero is a meaningful setting — it turns trust-modulated decay
+    # off, so every memory fades at the same rate regardless of where it came
+    # from. A greater-than bound made the one value that disables the feature
+    # the one value the API refused.
+    trust_kappa: Optional[float] = Field(None, ge=0.0)
     scheduler_interval_minutes: Optional[int] = Field(None, ge=1)
     core_memory_immune: Optional[StrictBool] = None
 
@@ -169,7 +173,7 @@ _FORGETTING_DEFAULTS: dict = {
 
 
 @router.get("/storage/config")
-def get_storage_config():
+def get_storage_config(request: Request = None):  # noqa: B008 - FastAPI idiom
     """Return current storage backend configuration.
 
     base_dir is read-only — it is derived from the process namespace and
@@ -177,9 +181,27 @@ def get_storage_config():
     """
     try:
         data = _read_config()
+        declared_graph = data.get("graph_backend", "auto")
+        declared_vector = data.get("vector_backend", "auto")
+        active_graph, active_vector = _active_backends(declared_graph, declared_vector)
         return {
-            "graph_backend": data.get("graph_backend", "auto"),
-            "vector_backend": data.get("vector_backend", "auto"),
+            "graph_backend": declared_graph,
+            "vector_backend": declared_vector,
+            # What is actually answering queries. A store can be configured for
+            # a backend it never successfully promoted to, and then the setting
+            # describes an intention rather than the system.
+            "graph_backend_active": active_graph,
+            "vector_backend_active": active_vector,
+            "backend_matches_configuration": (
+                active_graph == declared_graph and active_vector == declared_vector
+            ),
+            "scale_engine_state": data.get("scale_engine_state", "local_core"),
+            # What the last start did about moving this store onto the graph
+            # and vector backends. A store that could not be moved says why
+            # here rather than looking like one that was never asked.
+            "automatic_promotion": getattr(
+                request.app.state, "scale_autopromotion", None,
+            ) if request is not None else None,
             "base_dir": data.get("base_dir", str(MEMORY_DIR)),
         }
     except Exception:
@@ -187,6 +209,43 @@ def get_storage_config():
         return JSONResponse({"error": "Internal server error"}, status_code=500)
 
 
+def _active_backends(declared_graph: str, declared_vector: str) -> tuple[str, str]:
+    """Which backends are really serving queries, not which were requested.
+
+    A promotion writes the chosen backend into the configuration before the
+    directory that holds it exists, and a promotion that never completed leaves
+    the setting saying ``cozo`` while every query is answered by SQLite. The
+    dashboard read the setting, so it agreed with the mistake.
+
+    Resolved from the two things that must both be true for a backend to serve:
+    its library imports, and its data directory is on disk.
+    """
+    def usable(module: str, directory: str) -> bool:
+        # Importing it, not merely finding it. A package whose native extension
+        # no longer matches the interpreter is present on disk and raises on
+        # import, and "the file is there" would report it as serving queries it
+        # cannot answer.
+        import importlib
+
+        if not (MEMORY_DIR / directory).is_dir():
+            return False
+        try:
+            importlib.import_module(module)
+        except Exception:  # noqa: BLE001 - any import failure means unusable
+            return False
+        return True
+
+    graph = "sqlite"
+    if declared_graph in ("cozo", "auto") and usable("pycozo", "cozo"):
+        graph = "cozo"
+
+    vector = "sqlite-vec"
+    if declared_vector in ("lancedb", "auto") and usable("lancedb", "lance"):
+        vector = "lancedb"
+
+    return graph, vector
+
+
 # ---------------------------------------------------------------------------
 # PUT /api/v3/storage/config
 # ---------------------------------------------------------------------------
@@ -519,3 +578,126 @@ def put_graph_config(request: Request, body: GraphPruningConfigUpdate):
     except Exception:
         logger.exception("put_graph_config failed")
         return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+# ---------------------------------------------------------------------------
+# Ollama model selection
+# ---------------------------------------------------------------------------
+
+
+class OllamaModelCheck(BaseModel):
+    """A model a user is considering, and what they want to use it for."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    model_name: str = Field(..., min_length=1, max_length=200)
+    role: str = Field("embedding", pattern="^(embedding|generation)$")
+
+
+@router.get("/ollama/models")
+def get_ollama_models():
+    """Which Ollama models are installed, and which two are in use.
+
+    A user picking a model should be picking from a list, not typing a name and
+    finding out later that they typed it wrong.
+    """
+    from superlocalmemory.core.ollama_validator import DEFAULT_BASE_URL
+
+    try:
+        data = _read_config()
+        embedding = data.get("embedding") or {}
+        llm = data.get("llm") or {}
+        base_url = llm.get("base_url") or DEFAULT_BASE_URL
+
+        installed: list[dict] = []
+        reachable = True
+        detail = ""
+        try:
+            import httpx
+
+            response = httpx.get(f"{base_url.rstrip('/')}/api/tags", timeout=3.0)
+            if response.status_code == 200:
+                for entry in response.json().get("models", []):
+                    installed.append({
+                        "name": entry.get("name", ""),
+                        "size": entry.get("size", 0),
+                    })
+            else:
+                reachable = False
+                detail = f"Ollama answered {response.status_code}."
+        except Exception as exc:  # noqa: BLE001 - reported, not raised
+            reachable = False
+            detail = f"Ollama is not running at {base_url}. Start it with: ollama serve ({exc})"
+
+        return {
+            "reachable": reachable,
+            "detail": detail,
+            "base_url": base_url,
+            "installed": sorted(installed, key=lambda m: m["name"]),
+            "embedding_model": embedding.get("ollama_model", ""),
+            "generation_model": llm.get("model", "") if llm.get("provider") == "ollama" else "",
+            "stored_dimension": _stored_dimension(),
+        }
+    except Exception:
+        logger.exception("get_ollama_models failed")
+        return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+@router.post("/ollama/validate")
+def post_ollama_validate(request: Request, body: OllamaModelCheck):
+    """Ask the server to actually use the model, before anything is saved.
+
+    For the embedding role this also decides whether the switch is safe for this
+    store: two vector widths cannot be compared, and a store holding both
+    answers similarity questions with noise rather than failing.
+    """
+    _require_admin(request)
+    from superlocalmemory.core.ollama_validator import (
+        DEFAULT_BASE_URL,
+        EMBEDDING,
+        check_embedding_model_change,
+        validate_ollama_model,
+    )
+
+    try:
+        data = _read_config()
+        llm = data.get("llm") or {}
+        embedding = data.get("embedding") or {}
+        base_url = llm.get("base_url") or DEFAULT_BASE_URL
+
+        if body.role != EMBEDDING:
+            probe = validate_ollama_model(body.model_name, body.role, base_url=base_url)
+            return {
+                "ok": probe.ok,
+                "message": probe.message,
+                "role": body.role,
+                "model_name": body.model_name,
+                "dimension": probe.dimension,
+                "safe_to_apply": probe.ok,
+            }
+
+        decision = check_embedding_model_change(
+            body.model_name,
+            db_path=MEMORY_DIR / "memory.db",
+            current_model=embedding.get("ollama_model", "")
+                or embedding.get("model_name", ""),
+            base_url=base_url,
+        )
+        return {
+            "ok": decision.allowed,
+            "message": decision.message,
+            "role": body.role,
+            "model_name": body.model_name,
+            "dimension": decision.new_dimension,
+            "stored_dimension": decision.stored_dimension,
+            "safe_to_apply": decision.allowed,
+        }
+    except Exception:
+        logger.exception("post_ollama_validate failed")
+        return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+def _stored_dimension() -> int | None:
+    from superlocalmemory.core.ollama_validator import stored_embedding_dimension
+
+    return stored_embedding_dimension(MEMORY_DIR / "memory.db")