superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/server/routes/behavioral.py

@@ -29,6 +29,66 @@ _MAX_OUTCOME_FACT_IDS = 100
 _MAX_FACT_ID_LENGTH = 200
 
 
+def _outcome_insert(
+    conn,
+    *,
+    outcome_id: str,
+    profile: str,
+    fact_ids_json: str,
+    outcome: str,
+    context_json: str,
+    now_iso: str,
+    reward: float,
+    recall_query_id: str,
+) -> tuple[str, tuple]:
+    """Build the ``action_outcomes`` INSERT from the columns that exist.
+
+    ``reward``, ``settled``, ``settled_at`` and ``recall_query_id`` all arrive
+    with M006, which is a DEFERRED migration — it runs after engine init, not
+    during ``apply_all``. This handler has always named them unconditionally, so
+    on a store where M006 has not run it raised ``no such column`` and returned
+    ``success: false`` for a write that could perfectly well have been recorded
+    without them.
+
+    ``recall_query_id`` is the one that matters for learning: it is what lets a
+    bandit play be settled from this report. The column has existed since M006
+    and was never written — 0 of 162 rows on a live store carry one — so
+    every play fell through to the neutral 120-second default and 165 arms sat
+    at alpha == beta. Recording it is the join key that closes the loop.
+
+    Degrading per column rather than per statement means an unmigrated store
+    still records the outcome; it just cannot attribute it to a play, which is
+    the honest reduction in capability.
+    """
+    columns = ["outcome_id", "profile_id", "query", "fact_ids_json",
+               "outcome", "context_json", "timestamp"]
+    values: list = [outcome_id, profile, "", fact_ids_json, outcome,
+                    context_json, now_iso]
+    try:
+        present = {
+            row[1] for row in conn.execute("PRAGMA table_info(action_outcomes)")
+        }
+    except Exception:  # pragma: no cover — defensive
+        present = set()
+
+    for name, value in (
+        ("reward", reward),
+        ("settled", 1),
+        ("settled_at", now_iso),
+        ("recall_query_id", recall_query_id),
+    ):
+        if name in present:
+            columns.append(name)
+            values.append(value)
+
+    placeholders = ", ".join("?" * len(columns))
+    sql = (
+        f"INSERT INTO action_outcomes ({', '.join(columns)}) "
+        f"VALUES ({placeholders})"
+    )
+    return sql, tuple(values)
+
+
 class ReportOutcomeRequest(BaseModel):
     """Bounded explicit outcome payload accepted from dashboard clients."""
 
@@ -41,6 +101,11 @@ class ReportOutcomeRequest(BaseModel):
     outcome: Literal["success", "failure", "partial"]
     action_type: StrictStr = Field(default="other", max_length=80)
     context: StrictStr = Field(default="", max_length=1000)
+    #: Which recall this outcome judges, if the caller knows. Optional because
+    #: recall does not currently return its query_id to callers, so nothing can
+    #: supply one yet; when it is absent the settler falls back to overlap
+    #: between ``memory_ids`` and the memories a play recorded showing.
+    recall_query_id: StrictStr = Field(default="", max_length=64)
 
     @field_validator("memory_ids")
     @classmethod
@@ -394,16 +459,16 @@ def report_outcome(request: Request, data: ReportOutcomeRequest):
                 fact_ids=memory_ids,
             )
             conn.execute(
-                "INSERT INTO action_outcomes "
-                "(outcome_id, profile_id, query, fact_ids_json, outcome, "
-                " context_json, timestamp, reward, settled, settled_at) "
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)",
-                (
-                    outcome_id, profile, "",
-                    json.dumps(memory_ids),
-                    outcome,
-                    json.dumps(context_dict),
-                    now_iso, reward, now_iso,
+                *_outcome_insert(
+                    conn,
+                    outcome_id=outcome_id,
+                    profile=profile,
+                    fact_ids_json=json.dumps(memory_ids),
+                    outcome=outcome,
+                    context_json=json.dumps(context_dict),
+                    now_iso=now_iso,
+                    reward=reward,
+                    recall_query_id=data.recall_query_id,
                 ),
             )
 

package/src/superlocalmemory/server/routes/compliance.py

@@ -40,9 +40,33 @@ except ImportError:
     logger.info("V3 compliance engine not available")
 
 
+def _require_manage(request) -> None:
+    """Require MANAGE on the active workspace, where the workspace has roles.
+
+    A store with no roles configured has nobody to check against, and refusing
+    there would break every personal install; the loopback boundary is the
+    control in that case.
+    """
+    try:
+        from superlocalmemory.access.rbac import Permission
+        from superlocalmemory.server.rbac_enforce import require_permission
+
+        require_permission(request, Permission.MANAGE, profile=get_active_profile())
+    except ImportError:  # pragma: no cover -- roles are always present in-tree
+        return
+
+
 @router.get("/api/compliance/status")
-async def compliance_status():
+async def compliance_status(request: Request):
     """Get compliance engine status for active profile."""
+    # This returns which operations ran and what categories of memory exist.
+    # That is operational metadata about someone's store; it had no gate at all.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="compliance-read",
+    )
     if not COMPLIANCE_AVAILABLE:
         return {"available": False, "message": "Compliance engine not available"}
 
@@ -132,8 +156,43 @@ async def query_audit_trail(
         return {"available": False, "error": "Internal server error"}
 
 
+def _erasure_succeeded(result: dict) -> bool:
+    """One place that decides, so the surfaces cannot disagree.
+
+    The erasure reports two things, because they are two questions: whether the
+    data is gone, and whether that can be shown afterwards. This route used to
+    inline its own list of failure markers and the CLI used a different one, so
+    the API could say failure while the command line printed COMPLETE for the
+    same erasure.
+
+    Anything that leaves either question unanswered is a failure here. A caller
+    that wants the finer distinction reads ``erasure_complete`` and
+    ``erasure_provable`` from the body, which are both in it.
+    """
+    markers = (
+        "vector_store_failures",
+        "audit_completion_failed",
+        "audit_request_failed",
+        "receipt_persist_failed",
+        "table_delete_failures",
+        "code_graph_failed",
+        "fact_expansion_fts_failed",
+        "working_sets_failed",
+        "residue_recount_failed",
+        "backup_scan_failed",
+    )
+    if any(result.get(marker) for marker in markers):
+        return False
+    # Present on the profile path; absent on the entity path, where its absence
+    # must not be read as a failure.
+    for verdict in ("erasure_complete", "erasure_provable"):
+        if verdict in result and not result.get(verdict):
+            return False
+    return True
+
+
 @router.post("/api/compliance/retention-policy")
-async def create_retention_policy(data: dict):
+async def create_retention_policy(request: Request, data: dict):
     """Create a compliance retention policy.
 
     Body: {
@@ -144,6 +203,17 @@ async def create_retention_policy(data: dict):
         applies_to: dict (optional)
     }
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
 
@@ -183,8 +253,19 @@ async def create_retention_policy(data: dict):
 
 
 @router.delete("/api/compliance/retention-policy")
-async def delete_retention_policy(name: str = Query(...)):
+async def delete_retention_policy(request: Request, name: str = Query(...)):
     """Delete a retention policy by name for the active profile."""
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -202,12 +283,23 @@ async def delete_retention_policy(name: str = Query(...)):
 
 
 @router.post("/api/compliance/retention/enforce")
-async def enforce_retention():
+async def enforce_retention(request: Request):
     """Run all retention policies for the active profile now.
 
     Moves expired facts to their rule's terminal lifecycle zone (archive/
     tombstone) or counts them (notify). Soft-state only — never a raw delete.
     """
+    # A retention rule decides what is kept and for how long, so changing one
+    # is a persistent configuration change on someone else's data. It is gated
+    # on the same loopback-trusted boundary the audit read uses, and on MANAGE
+    # in a workspace that has roles -- it had neither.
+    from superlocalmemory.server.write_identity import require_http_mutation_actor
+
+    require_http_mutation_actor(
+        request, getattr(request.app.state, "daemon_descriptor", None),
+        actor_kind="retention-policy",
+    )
+    _require_manage(request)
     if not COMPLIANCE_AVAILABLE:
         return {"success": False, "error": "Compliance engine not available"}
     try:
@@ -302,13 +394,7 @@ async def gdpr_erase(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_profile(profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {"success": success, "active_profile": profile, **result}
     except Exception:
         logger.exception("gdpr_erase error")
@@ -361,13 +447,7 @@ async def gdpr_erase_entity(request: Request, data: dict = {}):
         result = GDPRCompliance(engine._db, engine=engine).forget_entity(entity_name, profile)
         authorization.complete()
         result = result or {}
-        failure_markers = (
-            "vector_store_failures",
-            "audit_completion_failed",
-            "audit_request_failed",
-            "receipt_persist_failed",
-        )
-        success = not any(result.get(marker) for marker in failure_markers)
+        success = _erasure_succeeded(result)
         return {
             "success": success, "active_profile": profile,
             "entity_name": entity_name, **result,

package/src/superlocalmemory/server/routes/config_api.py

@@ -136,7 +136,11 @@ class ForgettingConfigUpdate(BaseModel):
     forget_threshold: Optional[float] = Field(None, ge=0.0, le=1.0)
     learning_rate: Optional[float] = Field(None, gt=0.0)
     forgetting_drift_scale: Optional[float] = Field(None, gt=0.0)
-    trust_kappa: Optional[float] = Field(None, gt=0.0)
+    # ge, not gt: zero is a meaningful setting — it turns trust-modulated decay
+    # off, so every memory fades at the same rate regardless of where it came
+    # from. A greater-than bound made the one value that disables the feature
+    # the one value the API refused.
+    trust_kappa: Optional[float] = Field(None, ge=0.0)
     scheduler_interval_minutes: Optional[int] = Field(None, ge=1)
     core_memory_immune: Optional[StrictBool] = None
 
@@ -169,7 +173,7 @@ _FORGETTING_DEFAULTS: dict = {
 
 
 @router.get("/storage/config")
-def get_storage_config():
+def get_storage_config(request: Request = None):  # noqa: B008 - FastAPI idiom
     """Return current storage backend configuration.
 
     base_dir is read-only — it is derived from the process namespace and
@@ -177,9 +181,27 @@ def get_storage_config():
     """
     try:
         data = _read_config()
+        declared_graph = data.get("graph_backend", "auto")
+        declared_vector = data.get("vector_backend", "auto")
+        active_graph, active_vector = _active_backends(declared_graph, declared_vector)
         return {
-            "graph_backend": data.get("graph_backend", "auto"),
-            "vector_backend": data.get("vector_backend", "auto"),
+            "graph_backend": declared_graph,
+            "vector_backend": declared_vector,
+            # What is actually answering queries. A store can be configured for
+            # a backend it never successfully promoted to, and then the setting
+            # describes an intention rather than the system.
+            "graph_backend_active": active_graph,
+            "vector_backend_active": active_vector,
+            "backend_matches_configuration": (
+                active_graph == declared_graph and active_vector == declared_vector
+            ),
+            "scale_engine_state": data.get("scale_engine_state", "local_core"),
+            # What the last start did about moving this store onto the graph
+            # and vector backends. A store that could not be moved says why
+            # here rather than looking like one that was never asked.
+            "automatic_promotion": getattr(
+                request.app.state, "scale_autopromotion", None,
+            ) if request is not None else None,
             "base_dir": data.get("base_dir", str(MEMORY_DIR)),
         }
     except Exception:
@@ -187,6 +209,43 @@ def get_storage_config():
         return JSONResponse({"error": "Internal server error"}, status_code=500)
 
 
+def _active_backends(declared_graph: str, declared_vector: str) -> tuple[str, str]:
+    """Which backends are really serving queries, not which were requested.
+
+    A promotion writes the chosen backend into the configuration before the
+    directory that holds it exists, and a promotion that never completed leaves
+    the setting saying ``cozo`` while every query is answered by SQLite. The
+    dashboard read the setting, so it agreed with the mistake.
+
+    Resolved from the two things that must both be true for a backend to serve:
+    its library imports, and its data directory is on disk.
+    """
+    def usable(module: str, directory: str) -> bool:
+        # Importing it, not merely finding it. A package whose native extension
+        # no longer matches the interpreter is present on disk and raises on
+        # import, and "the file is there" would report it as serving queries it
+        # cannot answer.
+        import importlib
+
+        if not (MEMORY_DIR / directory).is_dir():
+            return False
+        try:
+            importlib.import_module(module)
+        except Exception:  # noqa: BLE001 - any import failure means unusable
+            return False
+        return True
+
+    graph = "sqlite"
+    if declared_graph in ("cozo", "auto") and usable("pycozo", "cozo"):
+        graph = "cozo"
+
+    vector = "sqlite-vec"
+    if declared_vector in ("lancedb", "auto") and usable("lancedb", "lance"):
+        vector = "lancedb"
+
+    return graph, vector
+
+
 # ---------------------------------------------------------------------------
 # PUT /api/v3/storage/config
 # ---------------------------------------------------------------------------
@@ -519,3 +578,126 @@ def put_graph_config(request: Request, body: GraphPruningConfigUpdate):
     except Exception:
         logger.exception("put_graph_config failed")
         return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+# ---------------------------------------------------------------------------
+# Ollama model selection
+# ---------------------------------------------------------------------------
+
+
+class OllamaModelCheck(BaseModel):
+    """A model a user is considering, and what they want to use it for."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    model_name: str = Field(..., min_length=1, max_length=200)
+    role: str = Field("embedding", pattern="^(embedding|generation)$")
+
+
+@router.get("/ollama/models")
+def get_ollama_models():
+    """Which Ollama models are installed, and which two are in use.
+
+    A user picking a model should be picking from a list, not typing a name and
+    finding out later that they typed it wrong.
+    """
+    from superlocalmemory.core.ollama_validator import DEFAULT_BASE_URL
+
+    try:
+        data = _read_config()
+        embedding = data.get("embedding") or {}
+        llm = data.get("llm") or {}
+        base_url = llm.get("base_url") or DEFAULT_BASE_URL
+
+        installed: list[dict] = []
+        reachable = True
+        detail = ""
+        try:
+            import httpx
+
+            response = httpx.get(f"{base_url.rstrip('/')}/api/tags", timeout=3.0)
+            if response.status_code == 200:
+                for entry in response.json().get("models", []):
+                    installed.append({
+                        "name": entry.get("name", ""),
+                        "size": entry.get("size", 0),
+                    })
+            else:
+                reachable = False
+                detail = f"Ollama answered {response.status_code}."
+        except Exception as exc:  # noqa: BLE001 - reported, not raised
+            reachable = False
+            detail = f"Ollama is not running at {base_url}. Start it with: ollama serve ({exc})"
+
+        return {
+            "reachable": reachable,
+            "detail": detail,
+            "base_url": base_url,
+            "installed": sorted(installed, key=lambda m: m["name"]),
+            "embedding_model": embedding.get("ollama_model", ""),
+            "generation_model": llm.get("model", "") if llm.get("provider") == "ollama" else "",
+            "stored_dimension": _stored_dimension(),
+        }
+    except Exception:
+        logger.exception("get_ollama_models failed")
+        return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+@router.post("/ollama/validate")
+def post_ollama_validate(request: Request, body: OllamaModelCheck):
+    """Ask the server to actually use the model, before anything is saved.
+
+    For the embedding role this also decides whether the switch is safe for this
+    store: two vector widths cannot be compared, and a store holding both
+    answers similarity questions with noise rather than failing.
+    """
+    _require_admin(request)
+    from superlocalmemory.core.ollama_validator import (
+        DEFAULT_BASE_URL,
+        EMBEDDING,
+        check_embedding_model_change,
+        validate_ollama_model,
+    )
+
+    try:
+        data = _read_config()
+        llm = data.get("llm") or {}
+        embedding = data.get("embedding") or {}
+        base_url = llm.get("base_url") or DEFAULT_BASE_URL
+
+        if body.role != EMBEDDING:
+            probe = validate_ollama_model(body.model_name, body.role, base_url=base_url)
+            return {
+                "ok": probe.ok,
+                "message": probe.message,
+                "role": body.role,
+                "model_name": body.model_name,
+                "dimension": probe.dimension,
+                "safe_to_apply": probe.ok,
+            }
+
+        decision = check_embedding_model_change(
+            body.model_name,
+            db_path=MEMORY_DIR / "memory.db",
+            current_model=embedding.get("ollama_model", "")
+                or embedding.get("model_name", ""),
+            base_url=base_url,
+        )
+        return {
+            "ok": decision.allowed,
+            "message": decision.message,
+            "role": body.role,
+            "model_name": body.model_name,
+            "dimension": decision.new_dimension,
+            "stored_dimension": decision.stored_dimension,
+            "safe_to_apply": decision.allowed,
+        }
+    except Exception:
+        logger.exception("post_ollama_validate failed")
+        return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+def _stored_dimension() -> int | None:
+    from superlocalmemory.core.ollama_validator import stored_embedding_dimension
+
+    return stored_embedding_dimension(MEMORY_DIR / "memory.db")

package/src/superlocalmemory/server/routes/data_io.py

@@ -12,7 +12,7 @@ import gzip
 import hashlib
 import json
 import logging
-from typing import Optional
+from typing import Any, Optional
 from datetime import datetime, timezone
 
 from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File
@@ -78,7 +78,22 @@ async def export_memories(
             use_v3 = False
 
         if use_v3:
+            # Withheld summaries are excluded, and this is about IMPORT, not
+            # tidiness. Import re-ingests every record through the normal
+            # pipeline, which mints a fresh memory with quarantined = 0 — so an
+            # export taken here and imported anywhere would resurrect all 1,195
+            # model-written rows as though the owner had written them, on a
+            # machine where nothing had gone wrong. The repair would then have
+            # to run again there.
+            #
+            # Nothing the owner wrote is lost: these are derived artefacts, the
+            # consolidator regenerates them from the facts that ARE exported,
+            # and their text is kept in consolidated_summaries. (That table is
+            # not in this export either — it is a view, not a memory. Worth
+            # revisiting when export covers derived state.)
             query = "SELECT * FROM atomic_facts WHERE profile_id = ?"
+            if _has_column(cursor, "atomic_facts", "quarantined"):
+                query += " AND COALESCE(quarantined, 0) = 0"
             params = [active_profile]
             if category:
                 query += " AND fact_type = ?"
@@ -154,6 +169,19 @@ async def export_memories(
         raise _internal_error("Export error")
 
 
+def _has_column(cursor: Any, table: str, column: str) -> bool:
+    """Whether ``table`` carries ``column`` in this database.
+
+    Presence-guarded because ``quarantined`` arrives with a migration and this
+    route must keep working on a store the engine has not opened.
+    """
+    try:
+        cursor.execute(f"PRAGMA table_info({table})")
+        return any(row[1] == column for row in cursor.fetchall())
+    except Exception:  # noqa: BLE001 -- an export must not fail over a probe
+        return False
+
+
 @router.post("/api/import")
 async def import_memories(request: Request, file: UploadFile = File(...)):
     """Import memories from JSON file using V3 engine."""

package/src/superlocalmemory/server/routes/entity.py

@@ -84,7 +84,19 @@ def list_entities(
 
     conn = get_read_connection(engine._config.db_path)
     try:
-        where = ["ce.profile_id = ?"]
+        # Hide the placeholder that dated facts with no recognised entity
+        # attach their temporal events to (core/store_pipeline.py,
+        # _ensure_unresolved_entity). It is a hook for a foreign key, not a
+        # concept, and it would otherwise appear here as a nameless entity and
+        # be counted in the total the owner reads as "entities I have".
+        #
+        # Only the presentation layer needs this. The other readers of
+        # canonical_entities are unaffected on inspection: graph_pruner uses
+        # `NOT IN (SELECT entity_id ...)` to find orphaned edges, where
+        # including it is correct; community_summary builds from graph edges and
+        # the placeholder has none; scale_engine counts for backend sync, not
+        # for display.
+        where = ["ce.profile_id = ?", "ce.entity_type != 'unresolved'"]
         params: list[object] = [profile]
         if entity_type and entity_type.lower() != "all":
             where.append("ce.entity_type = ? COLLATE NOCASE")