superlocalmemory 4.0.5 → 4.0.6

package/src/superlocalmemory/server/routes/brain.py

@@ -44,25 +44,31 @@ Design notes (LLD-04 §7 hard rules):
 
 from __future__ import annotations
 
+import json as _json
 import logging
+import shutil
 import sqlite3
+import subprocess
+import threading
+import time
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
 from fastapi import APIRouter, Depends, HTTPException, Request
+
 from superlocalmemory import __version__
 from superlocalmemory.brain import BrainTruthService
-
 from superlocalmemory.core.security_primitives import (
     redact_secrets,
     verify_install_token,
 )
+from superlocalmemory.infra.data_root import canonical_data_root
 from superlocalmemory.learning.database import LearningDatabase
 from superlocalmemory.learning.features import FEATURE_DIM
-from superlocalmemory.infra.data_root import canonical_data_root
-from superlocalmemory.storage.read_connection import ReadConnectionFactory
 from superlocalmemory.storage.agent_experience import get_profile_receipt_summary
+from superlocalmemory.storage.read_connection import ReadConnectionFactory
+
 from .helpers import get_active_profile
 
 logger = logging.getLogger("superlocalmemory.routes.brain")
@@ -84,6 +90,17 @@ _VERSION: str = __version__
 # NOTE: do NOT add the literal forbidden-key strings here — the U4 grep
 # guard runs over this file.
 
+# ---------------------------------------------------------------------------
+# Bounded-loops detection — TTL-cached subprocess probe.
+# The binary lives in a pipx venv; importlib.util.find_spec('bounded_loops')
+# will NOT find it from our venv.  Detect via PATH / filesystem instead.
+# ---------------------------------------------------------------------------
+
+_BL_PROBE: dict[str, object] = {}
+_BL_PROBE_LOCK = threading.Lock()
+_BL_PROBE_TTL = 60.0          # seconds between re-probes
+
+
 # Memory directory (home-dir based). Always resolved at call time so that
 # tests can override via monkeypatch on ``_learning_db_path``.
 _MEMORY_DIR_DEFAULT = None  # test-only compatibility override
@@ -596,7 +613,8 @@ def _adapter_last_sync_ago(adapter_name: str) -> int | None:
     honest empty rather than a fabricated number.
     """
     try:
-        from datetime import datetime as _dt, timezone as _tz
+        from datetime import datetime as _dt
+        from datetime import timezone as _tz
         memory_db = _memory_dir() / "memory.db"
         if not memory_db.exists():
             return None
@@ -620,6 +638,166 @@ def _adapter_last_sync_ago(adapter_name: str) -> int | None:
         return None
 
 
+def _detect_codex_config() -> dict:
+    """Detect Codex integration from ``~/.codex/config.toml``.
+
+    Evidence tier: CONFIGURED — the config file contains
+    ``[mcp_servers.superlocalmemory]``, which proves Codex is wired to use
+    SLM as an MCP server.  This is the same tier as Claude Code's install-
+    token check: file presence, not live traffic.
+
+    What we deliberately do NOT claim: that Codex is currently running, or
+    that the MCP connection has been used recently.  For live-traffic
+    evidence the correct signal is ``tool_events`` rows attributed to
+    ``SLM_AGENT_ID='codex'`` — a check not performed here because
+    ``tool_events`` has no ``agent_id`` column.
+    """
+    try:
+        codex_config = Path.home() / ".codex" / "config.toml"
+        if not codex_config.exists():
+            return {"active": False, "reason": "codex_config_absent"}
+        try:
+            content = codex_config.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return {"active": False, "reason": "codex_config_unreadable"}
+        if "[mcp_servers.superlocalmemory]" not in content:
+            return {"active": False, "reason": "no_slm_mcp_block"}
+        return {
+            "active": True,
+            "evidence_tier": "configured",
+            "evidence": "~/.codex/config.toml#[mcp_servers.superlocalmemory]",
+        }
+    except Exception:  # pragma: no cover — defensive
+        return {"active": False, "reason": "detection_error"}
+
+
+def _bounded_loops_probe_fresh() -> dict:
+    """Probe bounded-loops installation — may spawn one short subprocess.
+
+    Detection approach: ``shutil.which("bounded-loops-mcp")`` (PATH scan,
+    no subprocess), then ``bl --version`` with a 1.5 s timeout (see the note at the call
+    site: 300 ms sat too close to the measured ~226 ms and flickered).
+
+    Evidence tier: INSTALLED — the ``bounded-loops-mcp`` binary is on the
+    resolved PATH, meaning the pipx package is present.
+
+    What we do NOT claim: that the bridge contract ``bounded-loops.dev/
+    slm-bridge/v1`` is live or that any run has been observed.  A full
+    capability handshake requires an async MCP round-trip; that is outside
+    the scope of this synchronous probe.
+
+    Trap: do NOT use ``importlib.util.find_spec('bounded_loops')`` — the
+    package lives in its own pipx venv and is not importable from this one.
+    """
+    bl_mcp_path = shutil.which("bounded-loops-mcp")
+    if bl_mcp_path is None:
+        # Cross-check: ~/.bounded-loops data dir (pipx installs leave this).
+        bl_data = Path.home() / ".bounded-loops"
+        if bl_data.is_dir():
+            return {
+                "installed": True,
+                "version": None,
+                "bridge_contract": "bounded-loops.dev/slm-bridge/v1",
+                "evidence_tier": "data_dir",
+                "evidence": "~/.bounded-loops exists",
+                "note": "mcp binary not on PATH; data dir detected",
+                "is_real": True,
+                "source": "filesystem",
+            }
+        return {
+            "installed": False,
+            "reason": "bounded_loops_mcp_not_in_path",
+            "is_real": True,
+            "source": "shutil.which",
+        }
+
+    version: str | None = None
+    bl_path = shutil.which("bl")
+    if bl_path:
+        try:
+            # 1.5s, not 0.3s. Measured on this machine: a cold
+            # _compute_bounded_loops() takes ~226 ms end to end, most of it this
+            # subprocess — a 300 ms budget leaves ~74 ms of headroom, so under
+            # any load the probe times out and the card silently reports
+            # "Version: unknown" while `bl --version` prints "bl 0.6.4" from a
+            # shell. A version string that flickers with machine load is worse
+            # than no version string, because it looks like a real finding.
+            # The cost of widening is bounded: this runs in a worker thread via
+            # asyncio.to_thread and is TTL-cached for 60s, so the worst case is
+            # one 1.5s thread once a minute, never on the recall/remember path.
+            proc = subprocess.run(
+                [bl_path, "--version"],
+                capture_output=True,
+                text=True,
+                timeout=1.5,
+            )
+            raw = (proc.stdout.strip() or proc.stderr.strip())
+            if raw.startswith("bl "):
+                version = raw[3:].strip()
+        except Exception:  # pragma: no cover — timeout or missing binary
+            pass
+
+    return {
+        "installed": True,
+        "version": version,
+        "bridge_contract": "bounded-loops.dev/slm-bridge/v1",
+        "evidence_tier": "path",
+        "evidence": "shutil.which(bounded-loops-mcp)",
+        "note": "bridge capability requires a runtime MCP handshake; not probed here",
+        "is_real": True,
+        "source": "shutil.which + bl --version",
+    }
+
+
+def _compute_bounded_loops() -> dict:
+    """Section ``bounded_loops`` — installation/presence status.
+
+    Wraps ``_bounded_loops_probe_fresh`` in a 60-second TTL cache so the
+    subprocess call is never inline on every Brain request.  First call may
+    take up to 1.5 s (subprocess timeout); subsequent calls within the TTL
+    window return immediately from the cache. The probe is single-flight, so
+    concurrent callers share one subprocess rather than spawning one each.
+    """
+    with _BL_PROBE_LOCK:
+        cached = _BL_PROBE.get("result")
+        cached_at = float(_BL_PROBE.get("cached_at", 0.0))
+        if cached is not None and (time.monotonic() - cached_at) < _BL_PROBE_TTL:
+            return cached  # type: ignore[return-value]
+
+        # SINGLE-FLIGHT: run the probe while STILL HOLDING the lock.
+        # The previous shape released the lock between the check and the probe,
+        # so every concurrent caller that arrived after a TTL expiry spawned its
+        # own `bl --version` subprocess — N dashboard requests meant N processes,
+        # each up to 1.5s. Holding the lock makes the 2nd..Nth caller wait for
+        # the first result instead. Blocking here is safe: this whole function
+        # runs in a worker thread via asyncio.to_thread, so the event loop is
+        # never held, and it is off the recall/remember path entirely.
+        try:
+            fresh = _bounded_loops_probe_fresh()
+            fresh["section_enabled"] = bool(fresh.get("installed", False))
+        except Exception as exc:  # probe itself blew up
+            # Cache the FAILURE too, and label it as a failure rather than as a
+            # clean "not installed". Without caching it, every request would
+            # retry a broken probe forever; without labelling it, a broken probe
+            # is indistinguishable from Bounded Loops genuinely being absent —
+            # the silent-failure class this release exists to remove.
+            logger.warning("bounded-loops probe failed: %s", exc)
+            fresh = {
+                "installed": None,
+                "is_real": False,
+                "probe_failed": True,
+                "reason": f"probe_error:{type(exc).__name__}",
+                "section_enabled": False,
+                "source": "probe raised",
+            }
+
+        # Timestamp AFTER the probe so the TTL measures time since completion,
+        # not time since the request arrived.
+        _BL_PROBE["result"] = fresh
+        _BL_PROBE["cached_at"] = time.monotonic()
+        return fresh
+
+
 def _compute_cross_platform() -> dict:
     """Section ``cross_platform`` — live status per injection target.
 
@@ -700,26 +878,93 @@ def _compute_cross_platform() -> dict:
     out["mcp"] = {"active": True, "tool": "mcp__slm"}
     # CLI is trivially active on any install.
     out["cli"] = {"active": True}
+    # Codex: detect from ~/.codex/config.toml.
+    # evidence_tier="configured" — config proves SLM is wired; it does not
+    # prove recent MCP traffic (tool_events has no agent_id column).
+    out["codex"] = _detect_codex_config()
     return out
 
 
+def _registry_staleness() -> tuple[str, int | None]:
+    """Return (registry_status, newest_entry_seconds_ago).
+
+    Reads the raw session registry file to determine whether the registry
+    has been written to recently, regardless of profile matching.  This
+    distinguishes three silent-but-different states:
+
+    ``live``    — newest entry is within 2× the presence window (10 min)
+    ``stale``   — file exists, entries exist, but all are older than 10 min
+                  → TELEMETRY GAP: hooks exist but are not firing
+    ``empty``   — file exists but has no entries
+    ``absent``  — file does not exist (first-run or cleaned up)
+    ``unknown`` — file could not be read
+
+    IMPORTANT: the window is NOT widened here.  We report the gap honestly
+    rather than hiding it by using a larger cutoff on the query.
+    """
+    try:
+        from superlocalmemory.infra.data_root import state_path
+        registry_path = state_path(".active_sessions.json")
+        if not registry_path.exists():
+            return "absent", None
+        try:
+            raw = registry_path.read_text(encoding="utf-8")
+            data = _json.loads(raw)
+        except Exception:
+            return "unknown", None
+        if not isinstance(data, dict) or not data:
+            return "empty", None
+        now_ns = time.time_ns()
+        ts_vals = [
+            int(row.get("ts_ns", 0))
+            for row in data.values()
+            if isinstance(row, dict) and int(row.get("ts_ns", 0)) > 0
+        ]
+        if not ts_vals:
+            return "empty", None
+        newest_ns = max(ts_vals)
+        seconds_ago = max(0, int((now_ns - newest_ns) / 1_000_000_000))
+        # 2× the 300 s presence window = 600 s sane boundary.
+        if seconds_ago <= 600:
+            return "live", seconds_ago
+        return "stale", seconds_ago
+    except Exception:  # pragma: no cover — defensive
+        return "unknown", None
+
+
 def _compute_active_clients(profile_id: str) -> dict:
-    """Return recent host presence without leaking session identifiers.
+    """Return recent host presence with honest telemetry-gap detection.
 
     ``cross_platform`` describes configured integration targets.  This
-    separate read-model answers the different question users actually ask:
-    which host clients have recently interacted with this local brain?
+    separate read-model answers a different question: which host clients
+    have recently interacted with this brain?
+
+    Three distinct states are surfaced instead of a silent empty list:
+    ``NO_ACTIVITY``     — registry is live; no clients in the 5-min window
+    ``TELEMETRY_GAP``   — registry's newest entry is > 10 min old; hooks
+                          may be installed but are not writing presence
+    ``REGISTRY_ERROR``  — exception accessing the registry; reported as
+                          ``is_real: False`` rather than an empty success
     """
+    reg_status, newest_ago = _registry_staleness()
+
+    clients: list[dict] = []
+    registry_ok = True
     try:
         from superlocalmemory.hooks.session_registry import active_client_summary
         clients = active_client_summary(profile_id, within_seconds=300)
-    except Exception:
-        clients = []
+    except Exception as exc:  # distinguish failure from emptiness (Wave 4)
+        registry_ok = False
+        reg_status = "error"
+        logger.debug("active_clients: registry error: %s", exc)
+
     return {
-        "is_real": True,
+        "is_real": registry_ok,
         "scope": "profile",
         "window_seconds": 300,
         "clients": clients,
+        "registry_status": reg_status,
+        "newest_entry_seconds_ago": newest_ago,
         "source": "session_registry (ephemeral, profile-scoped)",
     }
 
@@ -1140,7 +1385,7 @@ async def get_brain(request: Request, profile_id: str | None = None) -> dict:
         preferences, learning, usage, bandit_snap, cache,
         cross_platform, outcomes_preview, evolution, active_clients,
         feedback_loop, source_quality, graph_summary, agent_experience,
-        brain_truth,
+        brain_truth, bounded_loops,
     ) = await asyncio.gather(
         asyncio.to_thread(_compute_preferences, profile_id),
         asyncio.to_thread(_compute_learning_status, profile_id, lrn_db),
@@ -1159,6 +1404,7 @@ async def get_brain(request: Request, profile_id: str | None = None) -> dict:
         asyncio.to_thread(_compute_graph_summary, profile_id),
         asyncio.to_thread(_compute_agent_experience, profile_id),
         asyncio.to_thread(truth_service.snapshot, profile_id),
+        asyncio.to_thread(_compute_bounded_loops),
         return_exceptions=True,
     )
 
@@ -1169,9 +1415,15 @@ async def get_brain(request: Request, profile_id: str | None = None) -> dict:
             return fallback
         return value
 
-    active_clients = _ok(active_clients, {"is_real": True, "scope": "profile", "clients": [],
-                                          "window_seconds": 300,
-                                          "source": "session_registry unavailable"})
+    active_clients = _ok(active_clients, {
+        "is_real": False,
+        "scope": "profile",
+        "clients": [],
+        "window_seconds": 300,
+        "registry_status": "error",
+        "newest_entry_seconds_ago": None,
+        "source": "session_registry unavailable",
+    })
     feedback_loop = _ok(feedback_loop, {"is_real": True, "signals_by_type": {},
                                          "explicit_signals": 0, "implicit_signals": 0,
                                          "settled_outcomes": 0, "mean_settled_reward": None,
@@ -1248,6 +1500,21 @@ async def get_brain(request: Request, profile_id: str | None = None) -> dict:
         "shadow_preview": _compute_shadow_preview(profile_id),
         "evolution_cost_preview": _compute_evolution_cost_preview(profile_id),
         "outcome_queue": _compute_outcome_queue_stats(profile_id),
+        # ``bounded_loops`` — lazy-probed, TTL-cached installation status.
+        # ``section_enabled`` drives the UI panel on/off decision.
+        # Evidence tier: path-based (shutil.which); bridge capability not probed.
+        # is_real=False and installed=None, NOT installed=False. If the probe
+        # task raised, we do not know whether Bounded Loops is installed — and
+        # saying "installed: False, is_real: True" asserts absence we never
+        # established. That reads to the UI, and to any API consumer, exactly
+        # like a machine with Bounded Loops genuinely not present.
+        "bounded_loops": _ok(bounded_loops, {
+            "is_real": False,
+            "installed": None,
+            "probe_failed": True,
+            "section_enabled": False,
+            "source": "bounded_loops_probe unavailable",
+        }),
         "meta": _meta_now(),
     }
 
@@ -1261,7 +1528,8 @@ def _compute_outcome_queue_stats(profile_id: str) -> dict:
     """
     try:
         from superlocalmemory.learning.outcome_queue import (
-            get_counters, queue_size,
+            get_counters,
+            queue_size,
         )
         counters = get_counters()
         qsz = queue_size()

package/src/superlocalmemory/server/routes/learning.py

@@ -30,6 +30,7 @@ from .learning_telemetry import (
     sqlite_status as _sqlite_status,
 )
 from superlocalmemory.storage.memory_write import memory_read
+from superlocalmemory.learning.engagement import derive_engagement_from_dbs
 
 logger = logging.getLogger("superlocalmemory.routes.learning")
 router = APIRouter()
@@ -338,31 +339,18 @@ def learning_status(request: Request):
             "signals": signal_count,
         }
 
-        # Engagement — v3.4.8: Fixed method name (was get_engagement_stats, actual is get_stats)
-        engagement = _get_engagement()
-        if engagement:
-            try:
-                stats = engagement.get_stats(active_profile)
-                health = engagement.get_health(active_profile)
-                active_days = stats.get("active_days", 0)
-                total_events = stats.get("total_events", 0)
-                memories_per_day = (
-                    round(total_events / active_days, 1) if active_days > 0 else 0
-                )
-                result["engagement"] = {
-                    "health_status": health.upper(),
-                    "days_active": active_days,
-                    "memories_per_day": memories_per_day,
-                    "total_events": total_events,
-                    "recall_count": stats.get("recall_count", 0),
-                    "store_count": stats.get("store_count", 0),
-                    "session_count": stats.get("session_count", 0),
-                    "engagement_score": stats.get("engagement_score", 0),
-                }
-            except Exception as exc:
-                logger.debug("engagement stats: %s", exc)
-                result["engagement"] = None
-        else:
+        # Engagement — v3.4.9: derive-on-read from existing tables (zero writes,
+        # satisfies I1 recall/remember latency, I3 no unbounded growth, I6 provenance).
+        # memory.db:atomic_facts drives store_count/days_active/health.
+        # learning.db:learning_signals drives recall_count (proxy: distinct queries).
+        try:
+            result["engagement"] = derive_engagement_from_dbs(
+                memory_db_path=MEMORY_DIR / "memory.db",
+                learning_db_path=LEARNING_DB,
+                profile_id=active_profile,
+            )
+        except Exception as exc:
+            logger.debug("derive_engagement_from_dbs: %s", exc)
             result["engagement"] = None
 
         # Tech preferences + workflow patterns from V3.1 behavioral store