superlocalmemory 3.6.22 → 3.7.0

package/src/superlocalmemory/server/write_identity.py

@@ -0,0 +1,147 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later
+
+"""Shared HTTP write-identity boundary.
+
+Caller-selected IDE or agent labels are audit metadata.  Authorization is
+derived only from the private capability for the exact daemon instance or the
+local install token used by the same-origin dashboard.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+from typing import Any
+
+from fastapi import HTTPException, Request
+
+
+def _header(request: Request, name: str) -> str:
+    headers = request.headers
+    value = headers.get(name, "")
+    if value:
+        return str(value)
+    lowered = name.lower()
+    for key, candidate in headers.items():
+        if str(key).lower() == lowered:
+            return str(candidate)
+    return ""
+
+
+def require_daemon_actor(request: Request, descriptor: Any | None) -> str:
+    """Require the private capability for one exact daemon instance."""
+    capability = _header(request, "X-SLM-Daemon-Capability")
+    if descriptor is None or not hmac.compare_digest(
+        capability,
+        str(descriptor.capability),
+    ):
+        raise HTTPException(403, detail="Invalid daemon capability")
+    target_instance = _header(request, "X-SLM-Target-Instance")
+    if not hmac.compare_digest(
+        target_instance,
+        str(descriptor.instance_id),
+    ):
+        raise HTTPException(409, detail="Daemon instance changed")
+    return f"daemon-capability:{descriptor.capability_fingerprint}"
+
+
+def require_write_actor(
+    request: Request,
+    descriptor: Any | None,
+    *,
+    actor_kind: str = "dashboard",
+) -> str:
+    """Return a capability-derived actor or reject the write."""
+    if _header(request, "X-SLM-Daemon-Capability"):
+        return require_daemon_actor(request, descriptor)
+
+    from superlocalmemory.core.security_primitives import verify_install_token
+
+    if verify_install_token(_header(request, "X-Install-Token")):
+        from superlocalmemory.core.engine_ingestion import local_trusted_actor_id
+
+        return local_trusted_actor_id(actor_kind)
+
+    from superlocalmemory.infra.auth_middleware import verify_api_key
+
+    api_key = _header(request, "X-SLM-API-Key")
+    if verify_api_key(api_key):
+        fingerprint = hashlib.sha256(
+            b"superlocalmemory-api-actor-v1\0" + api_key.encode("utf-8")
+        ).hexdigest()
+        return f"api-key:{actor_kind}:{fingerprint}"
+
+    raise HTTPException(403, detail="Authenticated write capability required")
+
+
+def require_http_mutation_actor(
+    request: Request,
+    descriptor: Any | None,
+    *,
+    actor_kind: str = "http-route",
+    mesh_secret: str | None = None,
+) -> str:
+    """Derive a principal for any state-changing HTTP operation.
+
+    Private credentials always take precedence.  An uncredentialed loopback
+    peer is the same local-user boundary as the filesystem capability and gets
+    a derived local actor.  Non-loopback callers fail closed.  Mesh credentials
+    are accepted only when the broker has an explicit shared secret.
+    """
+    credential_headers = (
+        "X-SLM-Daemon-Capability",
+        "X-Install-Token",
+        "X-SLM-API-Key",
+    )
+    if any(_header(request, name) for name in credential_headers):
+        return require_write_actor(
+            request,
+            descriptor,
+            actor_kind=actor_kind,
+        )
+
+    client_host = request.client.host if request.client else ""
+    is_test_client = (
+        client_host == "testclient"
+        and os.environ.get("SLM_TEST_ISOLATION") == "1"
+    )
+    if client_host in ("127.0.0.1", "::1", "localhost") or is_test_client:
+        from superlocalmemory.core.engine_ingestion import local_trusted_actor_id
+
+        return local_trusted_actor_id(actor_kind)
+
+    if mesh_secret:
+        presented = (
+            _header(request, "X-Mesh-Secret")
+            or _header(request, "Authorization").removeprefix("Bearer ").strip()
+        )
+        if presented and hmac.compare_digest(presented, mesh_secret):
+            fingerprint = hashlib.sha256(
+                b"superlocalmemory-mesh-actor-v1\0" + presented.encode("utf-8")
+            ).hexdigest()
+            return f"mesh-secret:{fingerprint}"
+
+    raise HTTPException(403, detail="Authenticated mutation capability required")
+
+
+def authenticated_request_actor(
+    request: Request,
+    descriptor: Any | None = None,
+    *,
+    actor_kind: str = "http-route",
+) -> str:
+    """Return the middleware-verified principal or verify route credentials."""
+    actor = getattr(request.state, "authenticated_actor", "")
+    if actor:
+        return str(actor)
+    return require_write_actor(request, descriptor, actor_kind=actor_kind)
+
+
+__all__ = [
+    "require_daemon_actor",
+    "authenticated_request_actor",
+    "require_http_mutation_actor",
+    "require_write_actor",
+]

package/src/superlocalmemory/storage/access_log.py

@@ -2,14 +2,15 @@
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3
 
-"""Access log for fact retrieval events.
+"""Exposure log for fact retrieval events.
 
 Tracks when facts are accessed (recall, auto_invoke, search).
-Used by Phase 2 auto-invoke for recency scoring (H1 fix).
+The historical table name is retained for compatibility.  A ``recall`` row is
+exposure telemetry only and must never be interpreted as positive feedback.
 All SQL parameterized (Rule 11). Silent errors (Rule 19).
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations

package/src/superlocalmemory/storage/database.py

@@ -130,7 +130,12 @@ class DatabaseManager:
         self.db_path = Path(db_path)
         self.db_path.parent.mkdir(parents=True, exist_ok=True)
         self._lock = threading.Lock()
-        self._txn_conn: sqlite3.Connection | None = None
+        # Transaction connections are thread-affine in sqlite3. A manager is
+        # shared across HTTP, materializer, and worker threads, so a process-
+        # global connection slot lets another thread accidentally execute on
+        # an uncommitted foreign connection. Keep the active connection local
+        # to the thread that owns the transaction.
+        self._txn_state = threading.local()
         self._enable_wal()
 
     def _enable_wal(self) -> None:
@@ -174,7 +179,7 @@ class DatabaseManager:
         """Atomic transaction. All writes commit or rollback together."""
         with self._lock:
             conn = self._connect()
-            self._txn_conn = conn
+            self._txn_state.conn = conn
             try:
                 yield
                 conn.commit()
@@ -182,7 +187,7 @@ class DatabaseManager:
                 conn.rollback()
                 raise
             finally:
-                self._txn_conn = None
+                self._txn_state.conn = None
                 conn.close()
 
     @contextmanager
@@ -196,7 +201,7 @@ class DatabaseManager:
         """
         with self._lock:
             conn = self._connect()
-            self._txn_conn = conn
+            self._txn_state.conn = conn
             try:
                 yield conn
                 conn.commit()
@@ -204,7 +209,7 @@ class DatabaseManager:
                 conn.rollback()
                 raise
             finally:
-                self._txn_conn = None
+                self._txn_state.conn = None
                 conn.close()
 
     def execute(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]:
@@ -212,8 +217,9 @@ class DatabaseManager:
 
         Uses shared conn inside transaction, else per-call with retry.
         """
-        if self._txn_conn is not None:
-            return self._txn_conn.execute(sql, params).fetchall()
+        transaction_conn = getattr(self._txn_state, "conn", None)
+        if transaction_conn is not None:
+            return transaction_conn.execute(sql, params).fetchall()
 
         last_error: Exception | None = None
         for attempt in range(_MAX_RETRIES):
@@ -438,6 +444,35 @@ class DatabaseManager:
             )
         return [self._row_to_fact(r) for r in rows]
 
+    def get_external_visible_facts(
+        self,
+        profile_id: str,
+        *,
+        include_global: bool = False,
+        include_shared: bool = False,
+    ) -> list[AtomicFact]:
+        """Cross-profile facts visible to ``profile_id`` under scope policy.
+
+        This is the bounded supplement used by profile-partitioned candidate
+        indexes.  It deliberately excludes the requester's own partition so a
+        fast local index can merge only the global/authorized-shared rows it
+        cannot discover itself.  The canonical scope predicate remains the
+        sole authorization rule.
+        """
+        if not include_global and not include_shared:
+            return []
+        where, params = _scope_where(
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
+        rows = self.execute(
+            f"SELECT * FROM atomic_facts WHERE {where} AND profile_id != ? "
+            "ORDER BY created_at DESC",
+            (*params, profile_id),
+        )
+        return [self._row_to_fact(r) for r in rows]
+
     _MAX_FACTS_PER_ENTITY_LOOKUP: int = 100
 
     def get_facts_by_entity(
@@ -755,11 +790,35 @@ class DatabaseManager:
             (fact_id, profile_id, json.dumps(tokens)),
         )
 
-    def get_all_bm25_tokens(self, profile_id: str) -> dict[str, list[str]]:
-        """Load full BM25 index: fact_id -> token list."""
+    def get_all_bm25_tokens(
+        self,
+        profile_id: str,
+        include_global: bool = False,
+        include_shared: bool = False,
+    ) -> dict[str, list[str]]:
+        """Load the visible legacy BM25 index: fact_id -> token list."""
+        if not include_global and not include_shared:
+            # Preserve the historical token-store contract, including repair
+            # tooling that can inspect orphaned token rows before facts exist.
+            rows = self.execute(
+                "SELECT fact_id, tokens FROM bm25_tokens WHERE profile_id = ?",
+                (profile_id,),
+            )
+            return {
+                dict(row)["fact_id"]: json.loads(dict(row)["tokens"])
+                for row in rows
+            }
+        where, params = _scope_where(
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+            prefix="af",
+        )
         rows = self.execute(
-            "SELECT fact_id, tokens FROM bm25_tokens WHERE profile_id = ?",
-            (profile_id,),
+            "SELECT bt.fact_id, bt.tokens FROM bm25_tokens AS bt "
+            "JOIN atomic_facts AS af ON af.fact_id = bt.fact_id "
+            f"WHERE {where}",
+            (*params,),
         )
         return {dict(r)["fact_id"]: json.loads(dict(r)["tokens"]) for r in rows}
 
@@ -1383,6 +1442,36 @@ class DatabaseManager:
         Retries 3x on SQLITE_BUSY (handled by execute()).
         All SQL parameterized (HR-05).
         """
+        from superlocalmemory.core.lifecycle_state import atomic_lifecycle_for
+
+        with self.transaction():
+            self._upsert_retention_in_transaction(
+                fact_id=fact_id,
+                profile_id=profile_id,
+                retention_score=retention_score,
+                memory_strength=memory_strength,
+                access_count=access_count,
+                last_accessed_at=last_accessed_at,
+                lifecycle_zone=lifecycle_zone,
+            )
+            self.execute(
+                "UPDATE atomic_facts SET lifecycle = ? "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (atomic_lifecycle_for(lifecycle_zone), fact_id, profile_id),
+            )
+
+    def _upsert_retention_in_transaction(
+        self,
+        *,
+        fact_id: str,
+        profile_id: str,
+        retention_score: float,
+        memory_strength: float,
+        access_count: int,
+        last_accessed_at: str,
+        lifecycle_zone: str,
+    ) -> None:
+        """Write one retention row using the caller's active transaction."""
         self.execute(
             "INSERT INTO fact_retention "
             "(fact_id, profile_id, retention_score, memory_strength, "
@@ -1409,9 +1498,11 @@ class DatabaseManager:
         Returns count of successfully upserted rows.
         """
         count = 0
+        from superlocalmemory.core.lifecycle_state import atomic_lifecycle_for
+
         with self.transaction():
             for f in facts:
-                self.upsert_retention(
+                self._upsert_retention_in_transaction(
                     fact_id=f["fact_id"],
                     profile_id=profile_id,
                     retention_score=f["retention"],
@@ -1420,6 +1511,11 @@ class DatabaseManager:
                     last_accessed_at=f["last_accessed_at"],
                     lifecycle_zone=f["zone"],
                 )
+                self.execute(
+                    "UPDATE atomic_facts SET lifecycle = ? "
+                    "WHERE fact_id = ? AND profile_id = ?",
+                    (atomic_lifecycle_for(f["zone"]), f["fact_id"], profile_id),
+                )
                 count += 1
         return count
 
@@ -1464,20 +1560,17 @@ class DatabaseManager:
             )
             return
 
-        # Update fact_retention
-        self.execute(
-            "UPDATE fact_retention SET lifecycle_zone = 'forgotten', "
-            "  retention_score = 0.0 "
-            "WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, profile_id),
-        )
+        from superlocalmemory.core.lifecycle_state import set_fact_lifecycle_zone
 
-        # Mark in atomic_facts as archived (valid enum value per A-CRIT-01)
-        self.execute(
-            "UPDATE atomic_facts SET lifecycle = 'archived' "
-            "WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, profile_id),
-        )
+        with self.transaction():
+            set_fact_lifecycle_zone(
+                self, [fact_id], "forgotten", profile_id=profile_id,
+            )
+            self.execute(
+                "UPDATE fact_retention SET retention_score = 0.0 "
+                "WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, profile_id),
+            )
 
     # ------------------------------------------------------------------
     # Phase E: CCQ Consolidated Blocks & Audit CRUD

package/src/superlocalmemory/storage/migration_runner.py

@@ -34,25 +34,64 @@ import sqlite3
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Iterable
 
 from superlocalmemory.storage.migrations import (
     M001_add_signal_features_columns as _M001,
+)
+from superlocalmemory.storage.migrations import (
     M002_model_state_history as _M002,
+)
+from superlocalmemory.storage.migrations import (
     M003_migration_log as _M003,
+)
+from superlocalmemory.storage.migrations import (
     M004_cross_platform_sync_log as _M004,
+)
+from superlocalmemory.storage.migrations import (
     M005_bandit_tables as _M005,
+)
+from superlocalmemory.storage.migrations import (
     M006_action_outcomes_reward as _M006,
+)
+from superlocalmemory.storage.migrations import (
     M007_pending_outcomes as _M007,
+)
+from superlocalmemory.storage.migrations import (
     M009_model_lineage as _M009,
+)
+from superlocalmemory.storage.migrations import (
     M010_evolution_config as _M010,
+)
+from superlocalmemory.storage.migrations import (
     M011_archive_and_merge as _M011,
+)
+from superlocalmemory.storage.migrations import (
     M012_shadow_observations as _M012,
+)
+from superlocalmemory.storage.migrations import (
     M013_bi_temporal_columns as _M013,
+)
+from superlocalmemory.storage.migrations import (
     M014_v345_scale_ready as _M014,
+)
+from superlocalmemory.storage.migrations import (
     M015_add_pinned_column as _M015,
+)
+from superlocalmemory.storage.migrations import (
     M016_add_scope_support as _M016,
 )
+from superlocalmemory.storage.migrations import (
+    M017_ccq_scope_column as _M017,
+)
+from superlocalmemory.storage.migrations import (
+    M018_ingestion_operations as _M018,
+)
+from superlocalmemory.storage.migrations import (
+    M019_derivation_lineage as _M019,
+)
+from superlocalmemory.storage.migrations import (
+    M020_model_state_integrity as _M020,
+)
 
 # Map migration name → module (used for the optional ``verify(conn)`` hook
 # that lets the runner detect "already applied" state when an idempotent
@@ -73,6 +112,10 @@ _MODULES = {
     _M014.NAME: _M014,
     _M015.NAME: _M015,
     _M016.NAME: _M016,
+    _M017.NAME: _M017,
+    _M018.NAME: _M018,
+    _M019.NAME: _M019,
+    _M020.NAME: _M020,
 }
 
 logger = logging.getLogger(__name__)
@@ -101,6 +144,10 @@ MIGRATIONS: list[Migration] = [
     # M009 extends learning_model_state (created by M002).
     Migration(name=_M009.NAME, db_target="learning", ddl=_M009.DDL,
               dependencies=(_M002.NAME,)),
+    # M020 owns post-release integrity repair. M002 remains byte-for-byte
+    # compatible with databases that recorded its historical DDL hash.
+    Migration(name=_M020.NAME, db_target="learning", ddl=_M020.DDL,
+              dependencies=(_M002.NAME,)),
     # M010 creates evolution_config + evolution_llm_cost_log (learning.db).
     Migration(name=_M010.NAME, db_target="learning", ddl=_M010.DDL,
               dependencies=(_M003.NAME,)),
@@ -111,6 +158,10 @@ MIGRATIONS: list[Migration] = [
     Migration(name=_M004.NAME, db_target="memory", ddl=_M004.DDL),
     # M007 creates pending_outcomes (memory.db, LLD-00 §1.2).
     Migration(name=_M007.NAME, db_target="memory", ddl=_M007.DDL),
+    # M018 is additive and independent of runtime-bootstrapped tables.
+    Migration(name=_M018.NAME, db_target="memory", ddl=_M018.DDL),
+    Migration(name=_M019.NAME, db_target="memory", ddl=_M019.DDL,
+              dependencies=(_M018.NAME,)),
     # M006 + M011 are deliberately NOT here — see DEFERRED_MIGRATIONS below.
 ]
 
@@ -139,6 +190,8 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # M016 adds scope and shared_with columns to 5 core tables for
     # multi-scope memory support (personal/global/shared).
     Migration(name=_M016.NAME, db_target="memory", ddl=_M016.DDL),
+    # M017 adds scope to the engine-bootstrapped CCQ consolidation table.
+    Migration(name=_M017.NAME, db_target="memory", ddl=_M017.DDL),
 ]
 
 
@@ -395,6 +448,25 @@ def _bootstrap_both_migration_logs(
     return failed, details
 
 
+def _bootstrap_learning_schema(learning_db: Path, *, dry_run: bool) -> str | None:
+    """Create the base learning tables before forward migrations extend them.
+
+    ``apply_all`` is called by the daemon before ``MemoryEngine`` exists.  A
+    blank first-install therefore has no ``learning_signals`` or
+    ``learning_model_state`` tables for M001/M002/M009 to alter.  The runner
+    owns this prerequisite so every caller has the same first-boot contract.
+    """
+    if dry_run:
+        return None
+    try:
+        from superlocalmemory.learning.database import LearningDatabase
+
+        LearningDatabase(learning_db)
+    except Exception as exc:  # noqa: BLE001 - retain runner's non-fatal API
+        return f"learning schema bootstrap failed: {type(exc).__name__}: {exc}"
+    return None
+
+
 def apply_all(
     learning_db: Path,
     memory_db: Path,
@@ -411,6 +483,17 @@ def apply_all(
     failed: list[str] = []
     details: dict[str, str] = {}
 
+    schema_error = _bootstrap_learning_schema(learning_db, dry_run=dry_run)
+    if schema_error is not None:
+        failed.append("learning_schema_bootstrap")
+        details["learning_schema_bootstrap"] = schema_error
+        return {
+            "applied": applied,
+            "skipped": skipped,
+            "failed": failed,
+            "details": details,
+        }
+
     # S9-W1 C3: unify the migration_log bootstrap across both DBs up-front.
     bs_failed, bs_details = _bootstrap_both_migration_logs(
         learning_db, memory_db, dry_run=dry_run,

package/src/superlocalmemory/storage/migration_v33.py

@@ -21,7 +21,7 @@ by create_all_tables(). This migration module detects their absence
 and creates them for databases that were created before SLM 3.3.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations

package/src/superlocalmemory/storage/migrations/M002_model_state_history.py

@@ -16,7 +16,6 @@ reaching the LightGBM deserialiser.
 
 from __future__ import annotations
 
-import hashlib
 import sqlite3
 
 NAME = "M002_model_state_history"
@@ -40,19 +39,16 @@ def verify(conn: sqlite3.Connection) -> bool:
     return _REQUIRED_COLS <= cols
 
 
-# S9-W1 M-DATA-01: prior version of this migration hardcoded ``is_active=1``
-# for every copied row. v3.4.19 mainline has ``UNIQUE(profile_id)`` which
-# guarantees one row per profile so the hardcode worked — but a non-mainline
-# dev-build could have multiple rows and the partial unique index created
-# after rebuild would fail transactionally. We now mark only the row with
-# the MAX(id) per profile as active; everything else becomes history.
+# IMPORTANT: this DDL shipped in V3.4.21.  Migration hashes are immutable
+# upgrade contracts, so later improvements belong in a new forward migration
+# (M020), never in this historical definition.
 DDL = """
 BEGIN IMMEDIATE;
 
 CREATE TABLE learning_model_state_new (
     id                INTEGER PRIMARY KEY AUTOINCREMENT,
     profile_id        TEXT NOT NULL,
-    model_version     TEXT NOT NULL DEFAULT '3.4.22',
+    model_version     TEXT NOT NULL DEFAULT '3.4.21',
     state_bytes       BLOB NOT NULL,
     bytes_sha256      TEXT NOT NULL DEFAULT '',
     trained_on_count  INTEGER NOT NULL DEFAULT 0,
@@ -65,14 +61,8 @@ CREATE TABLE learning_model_state_new (
 
 INSERT INTO learning_model_state_new
     (profile_id, state_bytes, is_active, trained_at, updated_at)
-SELECT lms.profile_id, lms.state_bytes,
-       CASE WHEN lms.id = (
-             SELECT MAX(lms2.id)
-             FROM learning_model_state lms2
-             WHERE lms2.profile_id = lms.profile_id
-           ) THEN 1 ELSE 0 END,
-       lms.updated_at, lms.updated_at
-FROM learning_model_state lms;
+SELECT profile_id, state_bytes, 1, updated_at, updated_at
+FROM learning_model_state;
 
 DROP TABLE learning_model_state;
 ALTER TABLE learning_model_state_new RENAME TO learning_model_state;
@@ -86,47 +76,3 @@ CREATE INDEX idx_model_profile_time
 
 COMMIT;
 """
-
-
-def post_ddl_hook(conn: sqlite3.Connection) -> None:
-    """S9-W1 H-DATA-01: backfill ``bytes_sha256`` for every row copied forward.
-
-    The DDL INSERT could not list ``bytes_sha256`` because SQLite cannot
-    call a Python function inside an ``executescript`` block unless the
-    function is registered beforehand, and registering a UDF mid-DDL is
-    fragile. Instead, we run one UPDATE pass after the DDL commits.
-
-    Without this backfill, ``model_cache._parse_row`` calls
-    ``verify_sha256(state_bytes, '')`` which raises IntegrityError, the
-    parser tombstones the cache entry, and EVERY 18,000+ user who had a
-    trained model on v3.4.19 loses usable learned-ranker state on upgrade.
-
-    The fix is safe: SHA-256 of the already-persisted blob is
-    deterministic, adds <1 ms per profile, and never alters
-    ``state_bytes``. Runs inside the same connection so any UPDATE error
-    surfaces to the runner as ``post_ddl_hook`` failed.
-    """
-    try:
-        rows = conn.execute(
-            "SELECT id, state_bytes FROM learning_model_state "
-            "WHERE bytes_sha256 = '' OR bytes_sha256 IS NULL"
-        ).fetchall()
-    except sqlite3.Error:
-        return  # table empty or schema not yet present — nothing to do.
-
-    if not rows:
-        return
-
-    updates = []
-    for row_id, state_bytes in rows:
-        if state_bytes is None:
-            continue
-        sha = hashlib.sha256(state_bytes).hexdigest()
-        updates.append((sha, row_id))
-
-    if updates:
-        conn.executemany(
-            "UPDATE learning_model_state SET bytes_sha256 = ? WHERE id = ?",
-            updates,
-        )
-        conn.commit()