superlocalmemory 3.7.8 → 3.8.1

package/src/superlocalmemory/server/write_identity.py

@@ -13,11 +13,29 @@ from __future__ import annotations
 import hashlib
 import hmac
 import os
+import sys
 from typing import Any
 
 from fastapi import HTTPException, Request
 
 
+# H-04 (3.7.9): the test-client auth bypass must be impossible in a real daemon
+# even if SLM_TEST_ISOLATION=1 leaks into a production environment. Anchor it to
+# pytest being present in the running interpreter — a production daemon process
+# never imports pytest — decided once at import time so it cannot be toggled per
+# request at runtime.
+_TEST_ISOLATION_ALLOWED = (
+    os.environ.get("SLM_TEST_ISOLATION") == "1" and "pytest" in sys.modules
+)
+
+# H-01 (3.7.9): by default a same-user loopback process is a trusted principal
+# and may write without a credential. Operators who want stricter isolation
+# (e.g. a shared machine, or binding beyond loopback) set
+# SLM_REQUIRE_CREDENTIALS=1 so that even loopback must present a daemon
+# capability, install token, or API key. Default OFF — unchanged behaviour.
+_REQUIRE_CREDENTIALS = os.environ.get("SLM_REQUIRE_CREDENTIALS") == "1"
+
+
 def _header(request: Request, name: str) -> str:
     headers = request.headers
     value = headers.get(name, "")
@@ -60,9 +78,21 @@ def require_write_actor(
     from superlocalmemory.core.security_primitives import verify_install_token
 
     if verify_install_token(_header(request, "X-Install-Token")):
-        from superlocalmemory.core.engine_ingestion import local_trusted_actor_id
-
-        return local_trusted_actor_id(actor_kind)
+        # The install token is embedded in the dashboard JS served over HTTP, so
+        # a LAN observer can read it. Accept it ONLY from a loopback caller (the
+        # local dashboard/CLI). A non-loopback holder must use an API key — this
+        # closes LAN privilege escalation via a leaked install token. In-process
+        # tests (testclient) keep the bypass only under SLM_TEST_ISOLATION.
+        client_host = request.client.host if request.client else ""
+        is_test_client = client_host == "testclient" and _TEST_ISOLATION_ALLOWED
+        # SEC-L-02: do NOT treat an empty client_host as loopback here — a
+        # missing peer address (e.g. behind a proxy that strips it) must not be
+        # trusted just because an install token is presented. require_http_mutation_actor
+        # already excludes "" from its loopback set; keep the two paths consistent.
+        if is_test_client or client_host in ("127.0.0.1", "::1", "localhost"):
+            from superlocalmemory.core.engine_ingestion import local_trusted_actor_id
+
+            return local_trusted_actor_id(actor_kind)
 
     from superlocalmemory.infra.auth_middleware import verify_api_key
 
@@ -103,11 +133,11 @@ def require_http_mutation_actor(
         )
 
     client_host = request.client.host if request.client else ""
-    is_test_client = (
-        client_host == "testclient"
-        and os.environ.get("SLM_TEST_ISOLATION") == "1"
-    )
-    if client_host in ("127.0.0.1", "::1", "localhost") or is_test_client:
+    is_test_client = client_host == "testclient" and _TEST_ISOLATION_ALLOWED
+    loopback = client_host in ("127.0.0.1", "::1", "localhost")
+    # H-01: the loopback trusted-actor bypass is suppressed when the operator
+    # opts into SLM_REQUIRE_CREDENTIALS; in-process tests keep the bypass.
+    if is_test_client or (loopback and not _REQUIRE_CREDENTIALS):
         from superlocalmemory.core.engine_ingestion import local_trusted_actor_id
 
         return local_trusted_actor_id(actor_kind)

package/src/superlocalmemory/storage/database.py

@@ -70,6 +70,17 @@ _MAX_RETRIES = _env_int("SLM_DB_MAX_RETRIES", 5)                # retry on SQLIT
 _RETRY_BASE_DELAY = _env_float("SLM_DB_RETRY_BASE_DELAY", 0.1)  # backoff base (s)
 
 
+def _unbounded_facts_ceiling() -> int:
+    """Hard upper bound applied when a fact fetch is called with limit=None, so
+    a large tenant can never materialize the whole table into memory. Tunable
+    via SLM_MAX_FACTS_UNBOUNDED (default 50_000)."""
+    import os as _os
+    try:
+        return max(1, int(_os.environ.get("SLM_MAX_FACTS_UNBOUNDED", "50000")))
+    except (TypeError, ValueError):
+        return 50000
+
+
 def _scope_where(
     profile_id: str,
     *,
@@ -147,6 +158,14 @@ class DatabaseManager:
             conn.commit()
         finally:
             conn.close()
+        # C4 encryption-at-rest defense-in-depth: sensitive DB files shipped
+        # world-readable (0644). Restrict to owner-only now that the file (and
+        # its WAL sidecars) exist. Best-effort, never blocks DB open.
+        try:
+            from superlocalmemory.core.security_primitives import harden_db_perms
+            harden_db_perms(self.db_path)
+        except Exception:
+            pass
 
     def initialize(self, schema_module: ModuleType) -> None:
         """Create all tables. *schema_module* must expose ``create_all_tables(conn)``."""
@@ -431,17 +450,16 @@ class DatabaseManager:
             include_global=include_global,
             include_shared=include_shared,
         )
-        if limit is not None:
-            rows = self.execute(
-                f"SELECT * FROM atomic_facts WHERE {where} "
-                "ORDER BY created_at DESC LIMIT ?",
-                (*params, int(limit)),
-            )
-        else:
-            rows = self.execute(
-                f"SELECT * FROM atomic_facts WHERE {where} ORDER BY created_at DESC",
-                (*params,),
-            )
+        # memory-bounding-02 + perf M-04: an unbounded fetch materializes the
+        # whole table (hundreds of MB with embeddings at 50k+ facts). Apply a
+        # hard, env-tunable ceiling even when the caller passes limit=None.
+        if limit is None:
+            limit = _unbounded_facts_ceiling()
+        rows = self.execute(
+            f"SELECT * FROM atomic_facts WHERE {where} "
+            "ORDER BY created_at DESC LIMIT ?",
+            (*params, int(limit)),
+        )
         return [self._row_to_fact(r) for r in rows]
 
     def get_external_visible_facts(
@@ -525,10 +543,19 @@ class DatabaseManager:
         "source_turn_ids_json", "session_id", "embedding",
         "fisher_mean", "fisher_variance", "lifecycle", "langevin_position",
         "emotional_valence", "emotional_arousal", "signal_type",
+        # Multi-scope (M016): allow re-scoping a fact after creation so a memory
+        # can be shared with a team or made global from the dashboard.
+        "scope", "shared_with",
     })
 
-    def update_fact(self, fact_id: str, updates: dict[str, Any]) -> None:
-        """Partial update on a fact. JSON-serializes list/dict values."""
+    def update_fact(self, fact_id: str, updates: dict[str, Any],
+                    profile_id: str | None = None) -> None:
+        """Partial update on a fact. JSON-serializes list/dict values.
+
+        Tenant safety: when ``profile_id`` is supplied the UPDATE is constrained
+        to that tenant, so a fact_id belonging to another profile cannot be
+        mutated. Authorized routes always pass it.
+        """
         if not updates:
             raise ValueError("updates dict must not be empty")
         bad_keys = set(updates) - self._UPDATABLE_FACT_COLUMNS
@@ -543,21 +570,44 @@ class DatabaseManager:
             else:
                 clean[k] = v
         set_clause = ", ".join(f"{k} = ?" for k in clean)
-        self.execute(
-            f"UPDATE atomic_facts SET {set_clause} WHERE fact_id = ?",
-            (*clean.values(), fact_id),
-        )
+        if profile_id is not None:
+            self.execute(
+                f"UPDATE atomic_facts SET {set_clause} WHERE fact_id = ? AND profile_id = ?",
+                (*clean.values(), fact_id, profile_id),
+            )
+        else:
+            self.execute(
+                f"UPDATE atomic_facts SET {set_clause} WHERE fact_id = ?",
+                (*clean.values(), fact_id),
+            )
 
-    def delete_fact(self, fact_id: str) -> None:
+    def delete_fact(self, fact_id: str, profile_id: str | None = None) -> None:
         """Hard-delete a fact.
 
         DatabaseManager connections enforce FKs (PRAGMA foreign_keys=ON), so
         embedding_metadata / fact_retention / edges cascade. The explicit
         embedding_metadata delete below is belt-and-suspenders for the case a
         future caller routes through a connection without FK enforcement.
+
+        Tenant safety: when ``profile_id`` is supplied the delete is constrained
+        to that tenant (the fact must belong to it), so a fact_id from another
+        profile cannot be destroyed. Authorized routes always pass it.
         """
+        if profile_id is not None:
+            row = self.execute(
+                "SELECT 1 FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+            if not row:
+                return  # not this tenant's fact — no-op
         self.execute("DELETE FROM embedding_metadata WHERE fact_id = ?", (fact_id,))
-        self.execute("DELETE FROM atomic_facts WHERE fact_id = ?", (fact_id,))
+        if profile_id is not None:
+            self.execute(
+                "DELETE FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+        else:
+            self.execute("DELETE FROM atomic_facts WHERE fact_id = ?", (fact_id,))
 
     def gc_orphaned_embedding_metadata(self) -> int:
         """Remove embedding_metadata rows whose parent atomic_fact is gone.
@@ -626,35 +676,55 @@ class DatabaseManager:
             fact_count=d["fact_count"],
         )
 
-    def store_alias(self, alias: EntityAlias) -> str:
-        """Persist an entity alias. Returns alias_id."""
+    def store_alias(self, alias: EntityAlias, profile_id: str) -> str:
+        """Persist an entity alias under a profile. Returns alias_id.
+
+        profile_id scopes the alias so the same entity_id appearing in two
+        profiles never shares aliases.
+        """
         self.execute(
             "INSERT OR REPLACE INTO entity_aliases "
-            "(alias_id, entity_id, alias, confidence, source) VALUES (?,?,?,?,?)",
-            (alias.alias_id, alias.entity_id, alias.alias,
+            "(alias_id, profile_id, entity_id, alias, confidence, source) "
+            "VALUES (?,?,?,?,?,?)",
+            (alias.alias_id, profile_id, alias.entity_id, alias.alias,
              alias.confidence, alias.source),
         )
         return alias.alias_id
 
-    def get_aliases_for_entity(self, entity_id: str) -> list[EntityAlias]:
-        """All aliases for a canonical entity."""
+    def get_aliases_for_entity(
+        self, entity_id: str, profile_id: str,
+    ) -> list[EntityAlias]:
+        """All aliases for a canonical entity within one profile."""
         rows = self.execute(
-            "SELECT * FROM entity_aliases WHERE entity_id = ?", (entity_id,),
+            "SELECT * FROM entity_aliases WHERE entity_id = ? AND profile_id = ?",
+            (entity_id, profile_id),
         )
         return [
             EntityAlias(**{k: dict(r)[k] for k in ("alias_id", "entity_id", "alias", "confidence", "source")})
             for r in rows
         ]
 
-    def get_memory_content_batch(self, memory_ids: list[str]) -> dict[str, str]:
-        """Batch-fetch original memory text. Returns {memory_id: content}."""
+    def get_memory_content_batch(
+        self, memory_ids: list[str], profile_id: str,
+        include_global: bool = False, include_shared: bool = False,
+    ) -> dict[str, str]:
+        """Batch-fetch original memory text. Returns {memory_id: content}.
+
+        C4 hardening: this exposes raw memory *content* and is reachable from
+        HTTP routes, so it is strictly tenant-scoped — a memory_id belonging to
+        another profile is never resolved. Widen only via the scope flags.
+        """
         if not memory_ids:
             return {}
         unique_ids = list(set(memory_ids))
         ph = ','.join('?' * len(unique_ids))
+        where, sparams = _scope_where(
+            profile_id, include_global=include_global, include_shared=include_shared,
+        )
         rows = self.execute(
-            f"SELECT memory_id, content FROM memories WHERE memory_id IN ({ph})",
-            tuple(unique_ids),
+            f"SELECT memory_id, content FROM memories "
+            f"WHERE memory_id IN ({ph}) AND {where}",
+            (*unique_ids, *sparams),
         )
         return {dict(r)["memory_id"]: dict(r)["content"] for r in rows}
 
@@ -876,11 +946,23 @@ class DatabaseManager:
     # Phase 0.6: Missing methods (BLOCKER / CRITICAL / HIGH)
     # ------------------------------------------------------------------
 
-    def get_fact(self, fact_id: str) -> AtomicFact | None:
-        """Get a single fact by ID."""
-        rows = self.execute(
-            "SELECT * FROM atomic_facts WHERE fact_id = ?", (fact_id,),
-        )
+    def get_fact(self, fact_id: str, profile_id: str | None = None) -> AtomicFact | None:
+        """Get a single fact by ID.
+
+        C4 defense-in-depth: when ``profile_id`` is provided the lookup is
+        tenant-scoped so a fact_id from another profile cannot resolve. Left
+        optional (fact_id is a random UUID sourced from already-scoped queries)
+        to avoid destabilizing the core store/consolidation write path.
+        """
+        if profile_id is not None:
+            rows = self.execute(
+                "SELECT * FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+        else:
+            rows = self.execute(
+                "SELECT * FROM atomic_facts WHERE fact_id = ?", (fact_id,),
+            )
         return self._row_to_fact(rows[0]) if rows else None
 
     def get_facts_by_ids(
@@ -1017,12 +1099,17 @@ class DatabaseManager:
                     out.setdefault(fid, []).append(scene)
         return out
 
-    def increment_entity_fact_count(self, entity_id: str) -> None:
-        """Atomically increment fact_count for a canonical entity."""
+    def increment_entity_fact_count(self, entity_id: str, profile_id: str = "default") -> None:
+        """Atomically increment fact_count for a canonical entity scoped to profile.
+
+        L-01 fix: the original query had no profile_id guard; any entity_id match
+        would be updated regardless of owner. The AND profile_id = ? clause prevents
+        cross-profile mutations via shared entity_id values.
+        """
         self.execute(
             "UPDATE canonical_entities SET fact_count = fact_count + 1 "
-            "WHERE entity_id = ?",
-            (entity_id,),
+            "WHERE entity_id = ? AND profile_id = ?",
+            (entity_id, profile_id),
         )
 
     def store_trust_score(self, ts: TrustScore) -> str:
@@ -1121,11 +1208,17 @@ class DatabaseManager:
             (fact_id, profile_id, contextual_description, keywords, generated_by),
         )
 
-    def get_fact_context(self, fact_id: str) -> dict | None:
-        """Get contextual description for a fact."""
-        rows = self.execute(
-            "SELECT * FROM fact_context WHERE fact_id = ?", (fact_id,),
-        )
+    def get_fact_context(self, fact_id: str, profile_id: str | None = None) -> dict | None:
+        """Get contextual description for a fact (C4: optionally tenant-scoped)."""
+        if profile_id is not None:
+            rows = self.execute(
+                "SELECT * FROM fact_context WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+        else:
+            rows = self.execute(
+                "SELECT * FROM fact_context WHERE fact_id = ?", (fact_id,),
+            )
         return dict(rows[0]) if rows else None
 
     def get_all_fact_contexts(self, profile_id: str) -> list[dict]:
@@ -1277,12 +1370,18 @@ class DatabaseManager:
             (fact_id, profile_id, valid_from, valid_until),
         )
 
-    def get_temporal_validity(self, fact_id: str) -> dict | None:
-        """Get temporal validity record for a fact."""
-        rows = self.execute(
-            "SELECT * FROM fact_temporal_validity WHERE fact_id = ?",
-            (fact_id,),
-        )
+    def get_temporal_validity(self, fact_id: str, profile_id: str | None = None) -> dict | None:
+        """Get temporal validity record for a fact (C4: optionally tenant-scoped)."""
+        if profile_id is not None:
+            rows = self.execute(
+                "SELECT * FROM fact_temporal_validity WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
+        else:
+            rows = self.execute(
+                "SELECT * FROM fact_temporal_validity WHERE fact_id = ?",
+                (fact_id,),
+            )
         return dict(rows[0]) if rows else None
 
     def get_all_temporal_validity(self, profile_id: str) -> list[dict]:
@@ -1316,18 +1415,106 @@ class DatabaseManager:
         """Get fact_ids that are currently valid (not expired).
 
         Returns facts that either have no temporal record (assumed valid)
-        or have valid_until IS NULL and system_expired_at IS NULL.
+        or whose temporal record satisfies:
+          - valid_until IS NULL (open-ended) OR valid_until > now() (still in window)
+          - system_expired_at IS NULL (not system-invalidated)
+
+        M-02 fix: the original query used ``tv.valid_until IS NULL`` which
+        incorrectly excluded future-dated (still valid) facts. The correct
+        predicate is a date comparison against the current timestamp.
         """
         rows = self.execute(
             "SELECT f.fact_id FROM atomic_facts f "
             "LEFT JOIN fact_temporal_validity tv ON f.fact_id = tv.fact_id "
             "WHERE f.profile_id = ? "
-            "  AND (tv.fact_id IS NULL OR tv.valid_until IS NULL) "
-            "  AND (tv.fact_id IS NULL OR tv.system_expired_at IS NULL)",
+            "  AND (tv.fact_id IS NULL "
+            "       OR ( "
+            "           (tv.valid_until IS NULL "
+            "            OR tv.valid_until > strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) "
+            "           AND tv.system_expired_at IS NULL "
+            "       ))",
             (profile_id,),
         )
         return [dict(r)["fact_id"] for r in rows]
 
+    def get_invalidated_fact_ids(
+        self, fact_ids: list[str], profile_id: str,
+    ) -> set[str]:
+        """Return the subset of ``fact_ids`` that are system-invalidated.
+
+        A fact is system-invalidated when ``system_expired_at`` is set — i.e.
+        it was superseded/contradicted by a newer fact (see
+        ``invalidate_fact_temporal``). Such facts are wrong/outdated and must be
+        excluded from default retrieval (T1, Phase 4).
+
+        Bounded + indexed: only the supplied candidate ids are queried (never a
+        full-table scan), keyed on the ``fact_id`` PK with the
+        ``idx_temporal_system_expired`` index covering the predicate. Chunked to
+        stay well under SQLite's ~999 bound-parameter limit. Facts with no
+        temporal record — or a record whose ``system_expired_at`` is NULL — are
+        NOT returned (treated as valid), so existing DBs need no backfill.
+
+        Event-time expiry (``valid_until`` in the past) is intentionally NOT
+        applied here: it is query-scoped (historical queries legitimately want
+        expired facts, per ``include_expired_in_history``) and handled by the
+        time-window path, not by this blanket admission filter.
+        """
+        if not fact_ids:
+            return set()
+        invalid: set[str] = set()
+        chunk = 900
+        for start in range(0, len(fact_ids), chunk):
+            batch = fact_ids[start:start + chunk]
+            placeholders = ",".join("?" for _ in batch)
+            rows = self.execute(
+                f"SELECT fact_id FROM fact_temporal_validity "
+                f"WHERE fact_id IN ({placeholders}) "
+                f"  AND profile_id = ? "
+                f"  AND system_expired_at IS NOT NULL",
+                (*batch, profile_id),
+            )
+            for r in rows:
+                invalid.add(dict(r)["fact_id"])
+        return invalid
+
+    def get_fact_event_times(
+        self, fact_ids: list[str], profile_id: str,
+    ) -> dict[str, str]:
+        """Map each candidate fact_id to its best-available event time.
+
+        Priority (most specific first): ``referenced_date`` (the date the fact
+        is *about*) → ``observation_date`` (when it was observed) →
+        ``valid_from`` (bi-temporal event start) → ``created_at`` (storage
+        time, always present). Used by time-window recall to prune candidates
+        by when the underlying event happened, falling back to capture time for
+        undated facts.
+
+        Bounded + indexed (candidate ids only, ``fact_id`` PK), chunked under
+        SQLite's bound-parameter limit. Facts absent from the result (unknown
+        id / wrong profile) are simply omitted.
+        """
+        if not fact_ids:
+            return {}
+        out: dict[str, str] = {}
+        chunk = 900
+        for start in range(0, len(fact_ids), chunk):
+            batch = fact_ids[start:start + chunk]
+            placeholders = ",".join("?" for _ in batch)
+            rows = self.execute(
+                f"SELECT f.fact_id AS fact_id, "
+                f"COALESCE(f.referenced_date, f.observation_date, "
+                f"         tv.valid_from, f.created_at) AS event_time "
+                f"FROM atomic_facts f "
+                f"LEFT JOIN fact_temporal_validity tv ON f.fact_id = tv.fact_id "
+                f"WHERE f.fact_id IN ({placeholders}) AND f.profile_id = ?",
+                (*batch, profile_id),
+            )
+            for r in rows:
+                d = dict(r)
+                if d.get("event_time"):
+                    out[d["fact_id"]] = d["event_time"]
+        return out
+
     def delete_temporal_validity(self, fact_id: str) -> None:
         """Delete temporal validity record (for testing/rollback only)."""
         self.execute(
@@ -1335,6 +1522,31 @@ class DatabaseManager:
             (fact_id,),
         )
 
+    # ------------------------------------------------------------------
+    # Phase 4 (T3b): fact-augmented key expansion (BM25 alt-keys)
+    # ------------------------------------------------------------------
+
+    def upsert_fact_expansion(self, fact_id: str, alt_keys: str) -> None:
+        """Store/replace a fact's alternate keys in ``fact_expansion_fts``.
+
+        Standalone FTS5 (no external-content triggers), so we replace by hand:
+        delete any prior row for the fact, then insert the new alt-keys. An
+        empty/blank ``alt_keys`` clears the fact's expansion entry. Fail-soft:
+        a missing FTS table (legacy DB) never breaks the write path.
+        """
+        try:
+            self.execute(
+                "DELETE FROM fact_expansion_fts WHERE fact_id = ?", (fact_id,)
+            )
+            if alt_keys and alt_keys.strip():
+                self.execute(
+                    "INSERT INTO fact_expansion_fts (fact_id, alt_keys) "
+                    "VALUES (?, ?)",
+                    (fact_id, alt_keys.strip()),
+                )
+        except Exception as exc:  # pragma: no cover — legacy/missing FTS table
+            logger.debug("upsert_fact_expansion skipped for %s: %s", fact_id, exc)
+
     # ------------------------------------------------------------------
     # Phase 5: Core Memory Blocks CRUD (Rule 15)
     # ------------------------------------------------------------------