superlocalmemory 3.6.23 → 3.7.0

package/src/superlocalmemory/core/derivation_lineage.py

@@ -0,0 +1,246 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Capture derivation lineage without inventing source spans."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def _lineage_id(profile_id: str, object_type: str, object_id: str, operation_id: str) -> str:
+    value = f"{profile_id}\0{object_type}\0{object_id}\0{operation_id}"
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _table_exists(db: Any, table: str) -> bool:
+    return bool(db.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,),
+    ))
+
+
+def _record(
+    db: Any,
+    *,
+    profile_id: str,
+    object_type: str,
+    object_id: str,
+    operation_id: str,
+    derivation_version: str,
+    source_status: str,
+    source_start: int | None = None,
+    source_end: int | None = None,
+    source_text_sha256: str = "",
+    source_fact_ids: tuple[str, ...] = (),
+    unresolved_reason: str = "",
+) -> None:
+    db.execute(
+        "INSERT OR REPLACE INTO derivation_lineage "
+        "(lineage_id,profile_id,object_type,object_id,operation_id,"
+        "derivation_version,source_status,source_start,source_end,"
+        "source_text_sha256,source_fact_ids_json,unresolved_reason) "
+        "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+        (
+            _lineage_id(profile_id, object_type, object_id, operation_id),
+            profile_id, object_type, object_id, operation_id,
+            derivation_version, source_status, source_start, source_end,
+            source_text_sha256,
+            json.dumps(list(source_fact_ids), separators=(",", ":")),
+            unresolved_reason,
+        ),
+    )
+
+
+def _fact_ids(value: Any) -> tuple[str, ...]:
+    try:
+        decoded = json.loads(value or "[]")
+    except (TypeError, ValueError):
+        return ()
+    if not isinstance(decoded, list):
+        return ()
+    return tuple(str(item) for item in decoded)
+
+
+def _capture_derived_rows(
+    db: Any,
+    *,
+    table: str,
+    id_column: str,
+    object_type: str,
+    profile_id: str,
+    operation_id: str,
+    derivation_version: str,
+    operation_fact_ids: frozenset[str],
+) -> None:
+    if not _table_exists(db, table):
+        return
+    rows = db.execute(
+        f'SELECT "{id_column}",fact_ids_json FROM "{table}" WHERE profile_id=?',
+        (profile_id,),
+    )
+    for row in rows:
+        source_ids = tuple(
+            fact_id
+            for fact_id in _fact_ids(row["fact_ids_json"])
+            if fact_id in operation_fact_ids
+        )
+        if not source_ids:
+            continue
+        _record(
+            db,
+            profile_id=profile_id,
+            object_type=object_type,
+            object_id=str(row[id_column]),
+            operation_id=operation_id,
+            derivation_version=derivation_version,
+            source_status="derived_from_facts",
+            source_fact_ids=source_ids,
+            unresolved_reason="direct_span_not_applicable",
+        )
+
+
+def capture_operation_lineage(
+    db: Any,
+    *,
+    operation_id: str,
+    profile_id: str,
+    raw_content: str,
+    fact_ids: tuple[str, ...],
+    derivation_version: str,
+) -> None:
+    """Persist operation lineage while distinguishing spans from derivations."""
+    if not _table_exists(db, "derivation_lineage"):
+        return
+    fact_lineage: dict[str, dict[str, Any]] = {}
+    for fact_id in fact_ids:
+        rows = db.execute(
+            "SELECT content FROM atomic_facts WHERE fact_id=? AND profile_id=?",
+            (fact_id, profile_id),
+        )
+        if not rows:
+            _record(
+                db, profile_id=profile_id, object_type="fact", object_id=fact_id,
+                operation_id=operation_id, derivation_version=derivation_version,
+                source_status="unresolved", unresolved_reason="final_fact_missing",
+            )
+            fact_lineage[fact_id] = {
+                "source_status": "unresolved",
+                "unresolved_reason": "final_fact_missing",
+            }
+            continue
+        content = str(rows[0]["content"])
+        start = raw_content.find(content)
+        if start >= 0:
+            _record(
+                db, profile_id=profile_id, object_type="fact", object_id=fact_id,
+                operation_id=operation_id, derivation_version=derivation_version,
+                source_status="exact", source_start=start,
+                source_end=start + len(content),
+                source_text_sha256=hashlib.sha256(content.encode("utf-8")).hexdigest(),
+            )
+            fact_lineage[fact_id] = {
+                "source_status": "exact",
+                "source_start": start,
+                "source_end": start + len(content),
+                "source_text_sha256": hashlib.sha256(
+                    content.encode("utf-8")
+                ).hexdigest(),
+            }
+        else:
+            _record(
+                db, profile_id=profile_id, object_type="fact", object_id=fact_id,
+                operation_id=operation_id, derivation_version=derivation_version,
+                source_status="unresolved",
+                unresolved_reason="no_exact_span_in_raw_source",
+            )
+            fact_lineage[fact_id] = {
+                "source_status": "unresolved",
+                "unresolved_reason": "no_exact_span_in_raw_source",
+            }
+
+    operation_fact_ids = frozenset(fact_ids)
+    _record(
+        db,
+        profile_id=profile_id,
+        object_type="profile",
+        object_id=profile_id,
+        operation_id=operation_id,
+        derivation_version=derivation_version,
+        source_status="not_applicable",
+        unresolved_reason="profile_scope_not_derived_from_source_span",
+    )
+    _capture_derived_rows(
+        db,
+        table="entity_profiles",
+        id_column="profile_entry_id",
+        object_type="entity_summary",
+        profile_id=profile_id,
+        operation_id=operation_id,
+        derivation_version=derivation_version,
+        operation_fact_ids=operation_fact_ids,
+    )
+    _capture_derived_rows(
+        db,
+        table="memory_scenes",
+        id_column="scene_id",
+        object_type="memory_scene",
+        profile_id=profile_id,
+        operation_id=operation_id,
+        derivation_version=derivation_version,
+        operation_fact_ids=operation_fact_ids,
+    )
+
+    if _table_exists(db, "graph_edges"):
+        rows = db.execute(
+            "SELECT edge_id,source_id,target_id FROM graph_edges WHERE profile_id=?",
+            (profile_id,),
+        )
+        for row in rows:
+            source_ids = tuple(
+                value
+                for value in (str(row["source_id"]), str(row["target_id"]))
+                if value in operation_fact_ids
+            )
+            if source_ids:
+                _record(
+                    db,
+                    profile_id=profile_id,
+                    object_type="graph_edge",
+                    object_id=str(row["edge_id"]),
+                    operation_id=operation_id,
+                    derivation_version=derivation_version,
+                    source_status="derived_from_facts",
+                    source_fact_ids=source_ids,
+                    unresolved_reason="direct_span_not_applicable",
+                )
+
+    if _table_exists(db, "bm25_tokens"):
+        indexed_ids = {
+            str(row["fact_id"])
+            for row in db.execute(
+                "SELECT fact_id FROM bm25_tokens WHERE profile_id=?", (profile_id,)
+            )
+        }
+        for fact_id in operation_fact_ids & indexed_ids:
+            lineage = fact_lineage.get(fact_id, {})
+            _record(
+                db,
+                profile_id=profile_id,
+                object_type="index_bm25",
+                object_id=fact_id,
+                operation_id=operation_id,
+                derivation_version=derivation_version,
+                source_status=str(lineage.get("source_status", "unresolved")),
+                source_start=lineage.get("source_start"),
+                source_end=lineage.get("source_end"),
+                source_text_sha256=str(lineage.get("source_text_sha256", "")),
+                source_fact_ids=(fact_id,),
+                unresolved_reason=str(
+                    lineage.get("unresolved_reason", "source_fact_lineage_missing")
+                ),
+            )
+
+
+__all__ = ["capture_operation_lineage"]

package/src/superlocalmemory/core/embedding_worker.py

@@ -24,6 +24,7 @@ from __future__ import annotations
 
 import json
 import os
+import platform
 import signal
 import sys
 
@@ -52,6 +53,18 @@ def _start_parent_watchdog() -> None:
     start_parent_watchdog()
 
 
+def _embedding_backend_order() -> tuple[str, str]:
+    """Return the stable local backend preference for this platform.
+
+    ONNX model export/loading can stall indefinitely on Apple Silicon while
+    the same cached model loads through PyTorch CPU in a few seconds.  Keep
+    ONNX first elsewhere and retain it as a fallback on macOS.
+    """
+    if sys.platform == "darwin" and platform.machine().lower() == "arm64":
+        return ("pytorch", "onnx")
+    return ("onnx", "pytorch")
+
+
 def _load_embedding_model(name: str) -> tuple:
     """Load embedding model. ONNX CPU-only first, PyTorch fallback.
 
@@ -59,26 +72,25 @@ def _load_embedding_model(name: str) -> tuple:
     """
     from sentence_transformers import SentenceTransformer
 
-    # ONNX with explicit CPU provider — avoids CoreML EP memory overhead.
-    try:
-        m = SentenceTransformer(
-            name,
-            backend="onnx",
-            trust_remote_code=True,
-            model_kwargs={"provider": "CPUExecutionProvider"},
-        )
-        return m, "onnx"
-    except Exception:
-        pass
-
-    # PyTorch CPU fallback.
-    try:
-        import torch
-        with torch.inference_mode():
-            m = SentenceTransformer(name, trust_remote_code=True, device="cpu")
-        return m, "pytorch"
-    except Exception:
-        return None, ""
+    for backend in _embedding_backend_order():
+        try:
+            if backend == "onnx":
+                m = SentenceTransformer(
+                    name,
+                    backend="onnx",
+                    trust_remote_code=True,
+                    model_kwargs={"provider": "CPUExecutionProvider"},
+                )
+            else:
+                import torch
+                with torch.inference_mode():
+                    m = SentenceTransformer(
+                        name, trust_remote_code=True, device="cpu",
+                    )
+            return m, backend
+        except Exception:
+            continue
+    return None, ""
 
 
 def _worker_main() -> None:

package/src/superlocalmemory/core/embeddings.py

@@ -61,12 +61,22 @@ class DimensionMismatchError(RuntimeError):
 # This lock is the secondary safety net for when the daemon isn't available.
 # ---------------------------------------------------------------------------
 
-_EMBEDDING_LOCK_FILE = Path.home() / ".superlocalmemory" / ".embedding.lock"
-_EMBEDDING_PID_FILE = Path.home() / ".superlocalmemory" / ".embedding-worker.pid"
 _MAX_CONCURRENT_WORKERS = int(os.environ.get("SLM_MAX_EMBEDDING_WORKERS", 1))
 _embedding_lock_fd: int | None = None
 
 
+def _embedding_lock_file() -> Path:
+    from superlocalmemory.infra.data_root import state_path
+
+    return state_path(".embedding.lock")
+
+
+def _embedding_pid_file() -> Path:
+    from superlocalmemory.infra.data_root import state_path
+
+    return state_path(".embedding-worker.pid")
+
+
 def _is_embedding_worker_alive() -> bool:
     """Check if an embedding worker PID file exists and that PID is alive.
 
@@ -74,21 +84,23 @@ def _is_embedding_worker_alive() -> bool:
     check if one is already running. Prevents duplicate 1.6GB workers.
     """
     try:
-        if not _EMBEDDING_PID_FILE.exists():
+        pid_file = _embedding_pid_file()
+        if not pid_file.exists():
             return False
-        pid = int(_EMBEDDING_PID_FILE.read_text().strip())
+        pid = int(pid_file.read_text().strip())
         os.kill(pid, 0)  # Signal 0 = check if alive
         return True
     except (ValueError, OSError, ProcessLookupError):
         # PID file invalid or process dead — clean up stale file
-        _EMBEDDING_PID_FILE.unlink(missing_ok=True)
+        _embedding_pid_file().unlink(missing_ok=True)
         return False
 
 
 def register_embedding_worker_pid(pid: int) -> None:
     """Write the embedding worker PID to the machine-wide PID file."""
-    _EMBEDDING_PID_FILE.parent.mkdir(parents=True, exist_ok=True)
-    _EMBEDDING_PID_FILE.write_text(str(pid))
+    pid_file = _embedding_pid_file()
+    pid_file.parent.mkdir(parents=True, exist_ok=True)
+    pid_file.write_text(str(pid))
 
 
 def acquire_embedding_lock(timeout: float = 5.0) -> bool:
@@ -108,10 +120,11 @@ def acquire_embedding_lock(timeout: float = 5.0) -> bool:
         return True  # No file locking on Windows — PID check above is the guard
 
     import fcntl
-    _EMBEDDING_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+    lock_file = _embedding_lock_file()
+    lock_file.parent.mkdir(parents=True, exist_ok=True)
 
     try:
-        _embedding_lock_fd = os.open(str(_EMBEDDING_LOCK_FILE), os.O_CREAT | os.O_RDWR)
+        _embedding_lock_fd = os.open(str(lock_file), os.O_CREAT | os.O_RDWR)
         deadline = time.time() + timeout
         while time.time() < deadline:
             try:
@@ -505,22 +518,45 @@ class EmbeddingService:
             self._worker_proc = None
 
     def _kill_worker(self) -> None:
-        """Terminate worker subprocess."""
+        """Terminate the worker and close every owned pipe exactly once."""
         if self._idle_timer is not None:
             self._idle_timer.cancel()
             self._idle_timer = None
-        if self._worker_proc is not None:
+
+        proc = self._worker_proc
+        if proc is not None:
+            # Detach first so re-entrant/finalizer cleanup is idempotent.
+            self._worker_proc = None
+            self._worker_ready = False
             try:
-                self._worker_proc.stdin.write('{"cmd":"quit"}\n')
-                self._worker_proc.stdin.flush()
-                self._worker_proc.wait(timeout=3)
+                proc.stdin.write('{"cmd":"quit"}\n')
+                proc.stdin.flush()
+                proc.wait(timeout=3)
             except Exception:
                 try:
-                    self._worker_proc.kill()
+                    returncode = proc.poll()
                 except Exception:
-                    pass
-            self._worker_proc = None
-            self._worker_ready = False
+                    returncode = None
+                # MagicMock/unknown poll results are treated conservatively as
+                # live; a real exited child always reports an integer code.
+                if returncode is None or not isinstance(returncode, int):
+                    try:
+                        proc.kill()
+                        proc.wait(timeout=3)
+                    except Exception:
+                        pass
+            finally:
+                # TextIOWrapper.close() can itself raise BrokenPipeError while
+                # flushing buffered stdin. Suppress it here, while the stream
+                # is still strongly referenced, so it cannot surface later as
+                # an unraisable finalizer warning.
+                for stream_name in ("stdin", "stdout", "stderr"):
+                    stream = getattr(proc, stream_name, None)
+                    if stream is not None:
+                        try:
+                            stream.close()
+                        except (BrokenPipeError, OSError, ValueError):
+                            pass
 
     def _reset_idle_timer(self) -> None:
         """Reset idle timer — kills worker after 2 min inactivity."""