superlocalmemory 4.0.5 → 4.0.6

package/src/superlocalmemory/learning/engagement.py

@@ -320,3 +320,168 @@ class EngagementTracker:
         if raw <= 0:
             return 0.0
         return raw / (raw + 20.0)
+
+
+# ---------------------------------------------------------------------------
+# Derive-on-read engagement — zero hot-path cost (Invariants I1, I3)
+# ---------------------------------------------------------------------------
+
+def derive_engagement_from_dbs(
+    memory_db_path: "Path | str",
+    learning_db_path: "Path | str",
+    profile_id: str,
+) -> "Dict[str, Any]":
+    """Derive engagement metrics from tables that already exist — no writes.
+
+    Source tables
+    -------------
+    memory.db : atomic_facts
+        store_count   — live facts (lifecycle in active/warm/cold)
+        days_active   — distinct calendar days with at least one live fact
+        recent_7d     — facts created in the last 7 days (drives health)
+    learning.db : learning_signals
+        recall_count  — COUNT(DISTINCT query) for the profile.
+                        This is a proxy: one distinct query = one recall
+                        session.  If the same query was run N times, it
+                        counts as 1.  The table is populated by the
+                        post_tool_outcome_hook when Claude Code surfaces
+                        recall results; it may be empty in environments
+                        without that hook, in which case recall_count = 0
+                        while store_count still reflects real activity.
+
+    Invariant compliance
+    --------------------
+    I1 — zero writes, no lock acquisition on the recall/remember hot path.
+    I3 — no new table or row is ever created; bounded by the existing
+         lifecycle-management and retention systems (atomic_facts rows age
+         through active → warm → cold → archived via Langevin dynamics;
+         learning_signals rows are pruned by the retention sweep).
+    I6 — every field carries a named source.
+
+    health_status (plain language, non-technical)
+    ----------------------------------------------
+    "ACTIVE"   — 10 or more memories added in the last 7 days
+    "WARM"     —  3–9 memories added in the last 7 days
+    "COLD"     —  1–2 memories added in the last 7 days
+    "INACTIVE" —  no new memories in the last 7 days
+    """
+    memory_db_path = Path(str(memory_db_path))
+    learning_db_path = Path(str(learning_db_path))
+
+    store_count: int = 0
+    days_active: int = 0
+    recent_7d: int = 0
+    recall_count: int = 0
+
+    # ── memory.db : atomic_facts ──────────────────────────────────────────
+    if memory_db_path.exists():
+        try:
+            # read-only URI; never creates the file, never acquires write lock
+            _uri = f"{memory_db_path.resolve().as_uri()}?mode=ro"
+            _conn = sqlite3.connect(_uri, uri=True, timeout=1.0)
+            _conn.execute("PRAGMA query_only=ON")
+            try:
+                _tables = {
+                    r[0]
+                    for r in _conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table'"
+                    ).fetchall()
+                }
+                if "atomic_facts" in _tables:
+                    # Only live facts (archived = soft-deleted / forgotten).
+                    # CRIT-fix-3: lifecycle filter avoids counting deleted facts
+                    # as "stored"; a profile with 1 000 facts all archived
+                    # should not report store_count=1000 to a non-technical user.
+                    _r = _conn.execute(
+                        "SELECT COUNT(*) FROM atomic_facts "
+                        "WHERE profile_id=? AND lifecycle IN ('active','warm','cold')",
+                        (profile_id,),
+                    ).fetchone()
+                    store_count = _r[0] if _r else 0
+
+                    _r = _conn.execute(
+                        "SELECT COUNT(DISTINCT SUBSTR(created_at, 1, 10)) "
+                        "FROM atomic_facts "
+                        "WHERE profile_id=? AND lifecycle IN ('active','warm','cold')",
+                        (profile_id,),
+                    ).fetchone()
+                    days_active = _r[0] if _r else 0
+
+                    _r = _conn.execute(
+                        "SELECT COUNT(*) FROM atomic_facts "
+                        "WHERE profile_id=? "
+                        "AND lifecycle IN ('active','warm','cold') "
+                        "AND created_at >= datetime('now', '-7 days')",
+                        (profile_id,),
+                    ).fetchone()
+                    recent_7d = _r[0] if _r else 0
+            finally:
+                _conn.close()
+        except Exception:
+            # Any failure (locked, missing, corrupt) → keep zeros;
+            # health stays INACTIVE which is the honest fallback.
+            pass
+
+    # ── learning.db : learning_signals (recall proxy) ─────────────────────
+    if learning_db_path.exists():
+        try:
+            _uri = f"{learning_db_path.resolve().as_uri()}?mode=ro"
+            _conn = sqlite3.connect(_uri, uri=True, timeout=1.0)
+            _conn.execute("PRAGMA query_only=ON")
+            try:
+                _tables = {
+                    r[0]
+                    for r in _conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table'"
+                    ).fetchall()
+                }
+                if "learning_signals" in _tables:
+                    # CRIT-fix-1: label is "distinct queries" not raw row count.
+                    # One query that surfaces 10 facts writes 10 rows; COUNT(*)
+                    # would inflate the number by 10×. COUNT(DISTINCT query)
+                    # approximates "sessions where you asked for something"
+                    # which is the intent of recall_count for non-technical users.
+                    _r = _conn.execute(
+                        "SELECT COUNT(DISTINCT query) FROM learning_signals "
+                        "WHERE profile_id=?",
+                        (profile_id,),
+                    ).fetchone()
+                    recall_count = _r[0] if _r else 0
+            finally:
+                _conn.close()
+        except Exception:
+            pass
+
+    # ── health derivation (plain language) ───────────────────────────────
+    if recent_7d >= _ACTIVE_THRESHOLD:
+        health_status = "ACTIVE"
+    elif recent_7d >= _WARM_THRESHOLD:
+        health_status = "WARM"
+    elif recent_7d >= 1:
+        health_status = "COLD"
+    else:
+        health_status = "INACTIVE"
+
+    total_events = store_count  # recalls add to proxy separately via recall_count
+    memories_per_day = (
+        round(store_count / days_active, 1) if days_active > 0 else 0
+    )
+    raw = (
+        0.4 * recall_count
+        + 0.3 * store_count
+        + 0.1 * days_active
+    )
+    score = (raw / (raw + 20.0)) if raw > 0 else 0.0
+
+    return {
+        "health_status": health_status,
+        "days_active": days_active,
+        "memories_per_day": memories_per_day,
+        "total_events": total_events,
+        "recall_count": recall_count,
+        "store_count": store_count,
+        "session_count": 0,          # not derivable without dedicated writes
+        "engagement_score": round(score, 4),
+        # I6 provenance — every figure is traceable to a named source table
+        "source": "memory.db:atomic_facts,learning.db:learning_signals",
+    }

package/src/superlocalmemory/mcp/tools_code_graph.py

@@ -204,8 +204,12 @@ def register_code_graph_tools(server, get_engine: Callable) -> None:
                 if fp in file_groups:
                     file_groups[fp][1].append(e)
 
-            for fp, (ns, es, fr) in file_groups.items():
-                store.store_file_nodes_edges(fp, ns, es, fr)
+            # Two-phase commit: all nodes first, then edges.
+            # This makes storage order-independent so cross-file CALLS edges
+            # (e.g., a.py calls bar() defined in b.py) are never silently
+            # dropped because of the file iteration order.
+            batch = [(fp, ns, es, fr) for fp, (ns, es, fr) in file_groups.items()]
+            store.commit_build_batch(batch)
 
             # Build in-memory graph
             engine = GraphEngine(store)
@@ -357,11 +361,14 @@ def register_code_graph_tools(server, get_engine: Callable) -> None:
                     continue
                 try:
                     source = full.read_bytes()
-                    file_nodes, file_edges = parser.parse_file(
+                    file_nodes, file_edges, file_import_map = parser.parse_file(
                         Path(fp), source, lang
                     )
                     import hashlib
                     from superlocalmemory.code_graph.models import FileRecord
+                    from superlocalmemory.code_graph.parser import (
+                        _clean_and_resolve_edges,
+                    )
                     fr = FileRecord(
                         file_path=fp,
                         content_hash=hashlib.sha256(source).hexdigest(),
@@ -371,7 +378,27 @@ def register_code_graph_tools(server, get_engine: Callable) -> None:
                         edge_count=len(file_edges),
                         last_indexed=time.time(),
                     )
-                    store.store_file_nodes_edges(fp, file_nodes, file_edges, fr)
+                    # Wire the resolver for the incremental path.
+                    # parse_all has _clean_and_resolve_edges built in, but
+                    # update_code_graph goes through parse_file which emits raw
+                    # placeholder targets (__call__<name>).  Without resolution,
+                    # Fix B (defensive filter) drops ALL CALLS edges — silent
+                    # data loss on every incremental update.
+                    #
+                    # Load the full DB node set as the resolution universe so
+                    # Strategy 3 (global heuristic) can match cross-file calls.
+                    db_nodes, _ = store.get_all_nodes_and_edges()
+                    resolution_universe = list(file_nodes) + [
+                        n for n in db_nodes if n.file_path != fp
+                    ]
+                    resolved_edges = _clean_and_resolve_edges(
+                        resolution_universe,
+                        list(file_edges),
+                        {fp: file_import_map},
+                        repo,
+                        config,
+                    )
+                    store.store_file_nodes_edges(fp, file_nodes, resolved_edges, fr)
                 except Exception as exc:
                     logger.warning("Failed to update %s: %s", fp, exc)
 

package/src/superlocalmemory/mcp/tools_v3.py

@@ -55,9 +55,12 @@ def register_v3_tools(server, get_engine: Callable) -> None:
     async def set_mode(mode: str) -> dict:
         """Switch operating mode (a, b, or c).
 
-        Mode A: Local Guardian (zero LLM, local embeddings only).
-        Mode B: Smart Local (local Ollama LLM, on-device inference).
-        Mode C: Full Power (configured cloud LLM provider, best accuracy).
+        Mode A (Local Guardian): Nothing leaves this device. No AI language
+                model runs. Fastest and most private.
+        Mode B: All data stays on this device. Uses a local Ollama AI model
+                to improve recall quality. Requires Ollama installed.
+        Mode C: Uses a cloud AI provider (OpenAI, Anthropic, …) for best
+                recall quality. Queries leave this device; API key required.
 
         Resets the engine to apply the new mode configuration.
 
@@ -377,8 +380,19 @@ def register_v3_tools(server, get_engine: Callable) -> None:
 def _mode_description(mode: str) -> str:
     """Human-readable capability description for a mode (never a legal claim)."""
     descriptions = {
-        "a": "Local Guardian: zero LLM, local embeddings only",
-        "b": "Smart Local: local Ollama LLM, on-device inference",
-        "c": "Full Power: configured cloud LLM provider, best accuracy",
+        "a": (
+            "Local Guardian — on-device only: no AI language model runs and "
+            "nothing leaves this device. Fastest and most private."
+        ),
+        "b": (
+            "Smart Local — on-device plus a local Ollama model: better recall "
+            "quality, and nothing leaves this device. Requires Ollama to be "
+            "installed and running."
+        ),
+        "c": (
+            "Full Power — uses a cloud AI provider (OpenAI, Anthropic, …) for "
+            "the best recall quality. Your queries leave this device and an "
+            "API key is required."
+        ),
     }
     return descriptions.get(mode, "Unknown mode")

package/src/superlocalmemory/retrieval/engine.py

@@ -1066,6 +1066,27 @@ class RetrievalEngine:
         if not applied:
             return fused, False, status
 
+        # The worker can report applied=True while returning scores=null — the
+        # subprocess answers, so the call "succeeded", but there is nothing to
+        # score with.  Iterating None here raised TypeError from OUTSIDE the
+        # try/except above (which only wraps the rerank call itself), so the
+        # error escaped into the recall path rather than degrading to the fused
+        # ordering.  Fail soft: reranking is a quality improvement on top of a
+        # correct result set, never a correctness requirement.
+        # `not scored` covers None AND an empty sequence. An empty list is the
+        # same defect wearing different clothes: the worker says applied=True but
+        # supplied nothing to rank with. Guarding only None would let [] through
+        # to build an empty score_map, and every candidate would then be scored
+        # against a degenerate min/max — silently shrinking the fused component
+        # by (1 - alpha) while still reporting the rerank as applied.
+        if not scored:
+            logger.warning(
+                "Cross-encoder worker reported applied=True with %s scores; "
+                "falling back to fused ranking for this query.",
+                "null" if scored is None else "empty",
+            )
+            return fused, False, "worker_null_scores"
+
         score_map = {fact.fact_id: score for fact, score in scored}
 
         # Min-max normalize CE scores to [0, 1] within the batch instead of

package/src/superlocalmemory/retrieval/remote_reranker.py

@@ -110,12 +110,30 @@ def is_remote_cross_encoder_backend(backend: str) -> bool:
     return (backend or "").strip().lower() in REMOTE_CROSS_ENCODER_BACKENDS
 
 
-def validate_remote_reranker_config(backend: str, endpoint: str) -> str | None:
+def validate_remote_reranker_config(
+    backend: str,
+    endpoint: str,
+    trust_plain_http_lan: bool = True,
+) -> str | None:
     """Return an actionable error string, or None when the pair is coherent.
 
     Covers the issue-#103 leftover directly: an endpoint configured against a
     LOCAL backend used to be dropped on the floor by ``SLMConfig.load``. It now
     produces a named error naming both keys and the exact edit to make.
+
+    Args:
+        backend:            Value of ``retrieval.cross_encoder_backend``.
+        endpoint:           Value of ``retrieval.cross_encoder_endpoint``.
+        trust_plain_http_lan: When True (the default), numeric RFC1918/ULA/
+            link-local addresses may use plain HTTP — the same security posture
+            as the local reranker, where memory text only crosses loopback.
+            Set to False in hardened deployments (zero-trust networks, shared
+            colocation) to require HTTPS for all non-loopback hosts.
+
+    Threat model note: trusting a private-LAN address does NOT prevent a
+    MITM attack by an adversary on the same physical LAN (e.g. via ARP
+    spoofing). This flag means "the LAN is under my control and I accept that
+    risk." It is not a claim that RFC1918 traffic is cryptographically secure.
     """
     backend = (backend or "").strip()
     endpoint = (endpoint or "").strip()
@@ -139,11 +157,23 @@ def validate_remote_reranker_config(backend: str, endpoint: str) -> str | None:
         )
     if not remote:
         return None
-    return _validate_endpoint_url(endpoint)
-
-
-def _validate_endpoint_url(endpoint: str) -> str | None:
-    """Scheme/host allow-listing for the operator-supplied rerank URL."""
+    return _validate_endpoint_url(endpoint, trust_plain_http_lan=trust_plain_http_lan)
+
+
+def _validate_endpoint_url(
+    endpoint: str,
+    trust_plain_http_lan: bool = True,
+) -> str | None:
+    """Scheme/host allow-listing for the operator-supplied rerank URL.
+
+    Plain-HTTP allowances (most-to-least trusted):
+      1. Loopback (127.x, ::1, localhost) — always allowed.
+      2. Numeric RFC1918/ULA/link-local addresses — allowed when
+         ``trust_plain_http_lan`` is True (the default).  Only numeric
+         addresses qualify; bare hostnames are never trusted because DNS is
+         mutable and not a trust boundary.
+      3. Everything else (public IPs, bare hostnames) — always requires HTTPS.
+    """
     try:
         parsed = urlparse(endpoint)
     except ValueError as exc:
@@ -177,11 +207,39 @@ def _validate_endpoint_url(endpoint: str) -> str | None:
             "retrieval.cross_encoder_api_key; it is sent as a Bearer header "
             "and never logged."
         )
-    if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname):
+    if parsed.scheme == "http":
+        hostname = parsed.hostname
+        if _is_loopback_host(hostname):
+            return None  # loopback always allowed regardless of trust flag
+        if trust_plain_http_lan and _is_private_lan_host(hostname):
+            # Numeric private address on an operator-trusted LAN. Threat model:
+            # an attacker on the same physical LAN can still MITM plain HTTP
+            # (ARP spoofing). This is allowed because the LAN is assumed to be
+            # under the operator's control. Set trust_plain_http_lan=False in
+            # hardened/zero-trust environments.
+            return None
+        if not _is_private_lan_host(hostname):
+            # Public IP, CGNAT, or a bare hostname (DNS not trusted as a
+            # proof of locality). Bare hostnames that happen to resolve to
+            # private IPs are NOT trusted: DNS can be poisoned or changed,
+            # so only provably-private numeric addresses are accepted.
+            return (
+                "retrieval.cross_encoder_endpoint must use HTTPS for this "
+                "host. Plain HTTP is allowed only for loopback "
+                "(127.x/::1/localhost) and numeric private-LAN addresses "
+                "(RFC1918: 10.x, 172.16-31.x, 192.168.x; IPv6 ULA fc00::/7; "
+                "link-local 169.254.x/fe80::). "
+                "Bare hostnames are not trusted even if they resolve to a "
+                "private IP — use a numeric address or configure HTTPS."
+            )
+        # Private-LAN address but trust_plain_http_lan is False (hardened mode)
         return (
-            "retrieval.cross_encoder_endpoint must use HTTPS for non-loopback "
-            "hosts because recall queries and candidate memory text cross this "
-            "connection. Plain HTTP is allowed only for localhost/loopback."
+            "retrieval.cross_encoder_endpoint uses plain HTTP to a "
+            "private-LAN address. HTTPS is required because "
+            "retrieval.trust_plain_http_lan is set to false. "
+            "Either configure a TLS-terminating proxy on the reranker, or "
+            "set retrieval.trust_plain_http_lan=true to permit plain HTTP "
+            "within your private network (default for new installs)."
         )
     return None
 
@@ -197,6 +255,42 @@ def _is_loopback_host(hostname: str) -> bool:
         return False
 
 
+def _is_private_lan_host(hostname: str) -> bool:
+    """True only for numeric private-range addresses (RFC1918, ULA, link-local).
+
+    Deliberate non-DNS: bare hostnames (e.g. ``my-reranker.lan``) return False
+    even if they currently resolve to a private IP. DNS is mutable and not a
+    trust boundary — an adversary who can influence DNS resolution can redirect
+    the endpoint to a public host, defeating the locality check. Only numeric
+    addresses are provably bound to a private range at configuration time.
+
+    Accepted ranges (Python 3.11+ ``ipaddress.is_private``):
+      IPv4  RFC1918: 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
+      IPv4  link-local: 169.254.0.0/16
+      IPv6  ULA: fc00::/7 (includes fd00::/8)
+      IPv6  link-local: fe80::/10
+
+    Excluded ranges (not accepted for plain HTTP):
+      CGNAT 100.64.0.0/10 — ISP-shared address space, not operator-controlled
+      172.15.0.0/8 and 172.32.0.0/8 — outside the 172.16.0.0/12 boundary
+      Public unicast addresses
+
+    IPv4-mapped IPv6 addresses (``::ffff:192.168.1.1``) are unwrapped to their
+    IPv4 equivalent before the range check, so they are handled consistently.
+    """
+    host = (hostname or "").rstrip(".").lower()
+    try:
+        addr = ipaddress.ip_address(host)
+    except ValueError:
+        # Not a numeric address — bare hostname, not provably private
+        return False
+    # Unwrap IPv4-mapped IPv6 (::ffff:192.168.1.1 → 192.168.1.1) so the
+    # RFC1918 check applies to the IPv4 portion.
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped
+    return addr.is_private
+
+
 def normalize_rerank_endpoint(endpoint: str) -> str:
     """Append ``/rerank`` when the URL stops at the API root.
 
@@ -370,8 +464,11 @@ class RemoteReranker:
         api_key: str = "",
         backend: str = "openai",
         timeout_seconds: float = _DEFAULT_READ_TIMEOUT_S,
+        trust_plain_http_lan: bool = True,
     ) -> None:
-        error = validate_remote_reranker_config(backend, endpoint)
+        error = validate_remote_reranker_config(
+            backend, endpoint, trust_plain_http_lan=trust_plain_http_lan,
+        )
         if error:
             raise RemoteRerankerConfigError(error)