superlocalmemory 3.8.13 → 4.0.0

package/src/superlocalmemory/core/transactions/service.py

@@ -0,0 +1,330 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from collections.abc import Iterable, Mapping
+
+from superlocalmemory.core.transactions.manifest import CompletionManifest
+from superlocalmemory.core.transactions.obligations import ObligationLedger
+from superlocalmemory.core.transactions.owners import (
+    ObligationKind,
+    ObligationState,
+    OperationContext,
+    ProjectionOwner,
+)
+from superlocalmemory.core.transactions.reconciler import Reconciler
+
+logger = logging.getLogger("superlocalmemory.core.transactions.service")
+
+MAX_APPLY_ATTEMPTS = 10
+
+
+class MemoryTransactionService:
+    def __init__(
+        self,
+        owners: Mapping[str, ProjectionOwner] | None = None,
+        *,
+        ledger: ObligationLedger | None = None,
+        reconciler: Reconciler | None = None,
+    ) -> None:
+        self._owners: dict[str, ProjectionOwner] = dict(owners or {})
+        self._ledger = ledger or ObligationLedger()
+        self._reconciler = reconciler or Reconciler(self._ledger)
+
+    @property
+    def owners(self) -> Mapping[str, ProjectionOwner]:
+        return dict(self._owners)
+
+    def register(self, owner: ProjectionOwner) -> None:
+        self._owners[owner.name] = owner
+
+    def record(
+        self,
+        conn: sqlite3.Connection,
+        context: OperationContext,
+        *,
+        owners: Iterable[str] | None = None,
+        kind: ObligationKind = ObligationKind.APPLY,
+    ) -> None:
+        names = list(owners) if owners is not None else list(self._owners)
+        self._ledger.record_many(conn, context, names, kind)
+
+    def apply(
+        self, conn: sqlite3.Connection, context: OperationContext,
+    ) -> None:
+        for obligation in self._ledger.fetch(conn, context.operation_id):
+            if obligation.kind is not ObligationKind.APPLY:
+                continue
+            if (
+                obligation.state in (ObligationState.FAILED, ObligationState.APPLIED)
+                and obligation.attempts >= MAX_APPLY_ATTEMPTS
+            ):
+                continue
+            self._reconcile_owner(conn, context, obligation)
+
+    def erase(
+        self, conn: sqlite3.Connection, context: OperationContext,
+    ) -> None:
+        for obligation in self._ledger.fetch(conn, context.operation_id):
+            if obligation.kind is not ObligationKind.ERASE:
+                continue
+            if obligation.state is ObligationState.ERASED:
+                continue
+            self._erase_one(conn, context, obligation.owner)
+
+    def compensate(
+        self, conn: sqlite3.Connection, context: OperationContext, owner_name: str,
+    ) -> None:
+        owner = self._owners.get(owner_name)
+        if owner is None:
+            self._ledger.mark(
+                conn, context.operation_id, owner_name, ObligationKind.APPLY,
+                ObligationState.FAILED,
+                detail={"phase": "compensate", "error": "owner not registered"},
+                bump_attempts=True,
+            )
+            return
+        try:
+            result = owner.compensate(context)
+        except Exception as exc:  # noqa: BLE001
+            self._ledger.mark(
+                conn, context.operation_id, owner_name, ObligationKind.APPLY,
+                ObligationState.FAILED,
+                detail={"phase": "compensate", "error": _err(exc)},
+                bump_attempts=True,
+            )
+            return
+        state = ObligationState.COMPENSATED if result.ok else ObligationState.FAILED
+        self._ledger.mark(
+            conn, context.operation_id, owner_name, ObligationKind.APPLY, state,
+            checksum=result.checksum,
+            detail={"phase": "compensate", **dict(result.detail)},
+            bump_attempts=True,
+        )
+
+    def reconcile(
+        self,
+        conn: sqlite3.Connection,
+        operation_id: str,
+        profile_id: str,
+        *,
+        canonical_committed: bool | None = None,
+    ) -> CompletionManifest:
+        return self._reconciler.reconcile(
+            conn, operation_id, profile_id,
+            canonical_committed=canonical_committed,
+        )
+
+    def run(
+        self,
+        conn: sqlite3.Connection,
+        context: OperationContext,
+        *,
+        canonical_committed: bool | None = None,
+    ) -> CompletionManifest:
+        self.apply(conn, context)
+        return self.reconcile(
+            conn, context.operation_id, context.profile_id,
+            canonical_committed=canonical_committed,
+        )
+
+    def reconcile_operation(
+        self,
+        db: object,
+        context: OperationContext,
+        *,
+        canonical_committed: bool | None = None,
+    ) -> CompletionManifest:
+        op = context.operation_id
+        with db.raw_connection() as conn:
+            obligations = self._ledger.fetch(conn, op)
+        for obligation in obligations:
+            if obligation.kind is not ObligationKind.APPLY:
+                continue
+            if (
+                obligation.state in (ObligationState.FAILED, ObligationState.APPLIED)
+                and obligation.attempts >= MAX_APPLY_ATTEMPTS
+            ):
+                continue
+            self._reconcile_owner_unlocked(db, context, obligation)
+        with db.raw_connection() as conn:
+            return self._reconciler.reconcile(
+                conn, op, context.profile_id,
+                canonical_committed=canonical_committed,
+            )
+
+    def _reconcile_owner_unlocked(
+        self, db: object, context: OperationContext, obligation: object,
+    ) -> None:
+        op = context.operation_id
+        owner_name = obligation.owner
+        owner = self._owners.get(owner_name)
+        if owner is None:
+            with db.raw_connection() as conn:
+                self._ledger.mark(
+                    conn, op, owner_name, ObligationKind.APPLY,
+                    ObligationState.FAILED,
+                    detail={"phase": "apply", "error": "owner not registered"},
+                    bump_attempts=True,
+                )
+            return
+        ok, checksum, detail = self._safe_verify(owner, context)
+        if (
+            ok
+            and obligation.state is ObligationState.VERIFIED
+            and obligation.checksum is not None
+            and checksum != obligation.checksum
+        ):
+            ok = False
+            detail = {"error": "projection drift: content changed since verification"}
+        if not ok:
+            applied_ok, a_checksum, a_detail = self._safe_apply(owner, context)
+            with db.raw_connection() as conn:
+                if applied_ok:
+                    self._ledger.mark(
+                        conn, op, owner_name, ObligationKind.APPLY,
+                        ObligationState.APPLIED, checksum=a_checksum,
+                        bump_attempts=True,
+                    )
+                else:
+                    self._ledger.mark(
+                        conn, op, owner_name, ObligationKind.APPLY,
+                        ObligationState.FAILED, checksum=a_checksum,
+                        detail={"phase": "apply", **a_detail}, bump_attempts=True,
+                    )
+            if applied_ok:
+                ok, checksum, detail = self._safe_verify(owner, context)
+        with db.raw_connection() as conn:
+            if ok:
+                updated = self._ledger.mark(
+                    conn, op, owner_name, ObligationKind.APPLY,
+                    ObligationState.VERIFIED, checksum=checksum,
+                    bump_verify_attempts=True, set_verified_at=True,
+                )
+            else:
+                updated = self._ledger.mark(
+                    conn, op, owner_name, ObligationKind.APPLY,
+                    ObligationState.FAILED, checksum=checksum,
+                    detail={"phase": "verify", **detail},
+                    bump_verify_attempts=True,
+                )
+        if updated == 0:
+            logger.warning(
+                "obligation %s/%s missing during reconcile mark", op, owner_name,
+            )
+
+    @staticmethod
+    def _safe_verify(
+        owner: ProjectionOwner, context: OperationContext,
+    ) -> tuple[bool, str | None, dict]:
+        try:
+            result = owner.verify(context)
+        except Exception as exc:  # noqa: BLE001
+            return False, None, {"error": _err(exc)}
+        return result.ok, result.checksum, dict(result.detail)
+
+    @staticmethod
+    def _safe_apply(
+        owner: ProjectionOwner, context: OperationContext,
+    ) -> tuple[bool, str | None, dict]:
+        try:
+            result = owner.apply(context)
+        except Exception as exc:  # noqa: BLE001
+            return False, None, {"error": _err(exc)}
+        return result.ok, result.checksum, dict(result.detail)
+
+    def verify_manifest(
+        self, conn: sqlite3.Connection, operation_id: str,
+    ) -> bool:
+        return self._reconciler.verify_manifest(conn, operation_id)
+
+    def fetch_manifest(
+        self, conn: sqlite3.Connection, operation_id: str,
+    ) -> CompletionManifest | None:
+        return self._reconciler.fetch_manifest(conn, operation_id)
+
+    def _reconcile_owner(
+        self, conn: sqlite3.Connection, context: OperationContext, obligation: object,
+    ) -> None:
+        op = context.operation_id
+        owner_name = obligation.owner
+        owner = self._owners.get(owner_name)
+        if owner is None:
+            self._ledger.mark(
+                conn, op, owner_name, ObligationKind.APPLY, ObligationState.FAILED,
+                detail={"phase": "apply", "error": "owner not registered"},
+                bump_attempts=True,
+            )
+            return
+        ok, checksum, detail = self._safe_verify(owner, context)
+        if (
+            ok
+            and obligation.state is ObligationState.VERIFIED
+            and obligation.checksum is not None
+            and checksum != obligation.checksum
+        ):
+            ok = False
+            detail = {"error": "projection drift: content changed since verification"}
+        if not ok:
+            applied_ok, a_checksum, a_detail = self._safe_apply(owner, context)
+            if applied_ok:
+                self._ledger.mark(
+                    conn, op, owner_name, ObligationKind.APPLY,
+                    ObligationState.APPLIED, checksum=a_checksum, bump_attempts=True,
+                )
+                ok, checksum, detail = self._safe_verify(owner, context)
+            else:
+                self._ledger.mark(
+                    conn, op, owner_name, ObligationKind.APPLY, ObligationState.FAILED,
+                    checksum=a_checksum, detail={"phase": "apply", **a_detail},
+                    bump_attempts=True,
+                )
+        if ok:
+            self._ledger.mark(
+                conn, op, owner_name, ObligationKind.APPLY, ObligationState.VERIFIED,
+                checksum=checksum, bump_verify_attempts=True, set_verified_at=True,
+            )
+        else:
+            self._ledger.mark(
+                conn, op, owner_name, ObligationKind.APPLY, ObligationState.FAILED,
+                checksum=checksum, detail={"phase": "verify", **detail},
+                bump_verify_attempts=True,
+            )
+
+    def _erase_one(
+        self, conn: sqlite3.Connection, context: OperationContext, owner_name: str,
+    ) -> None:
+        owner = self._owners.get(owner_name)
+        op = context.operation_id
+        if owner is None:
+            self._ledger.mark(
+                conn, op, owner_name, ObligationKind.ERASE, ObligationState.FAILED,
+                detail={"phase": "erase", "error": "owner not registered"},
+                bump_attempts=True,
+            )
+            return
+        try:
+            proof = owner.erase(context)
+        except Exception as exc:  # noqa: BLE001
+            self._ledger.mark(
+                conn, op, owner_name, ObligationKind.ERASE, ObligationState.FAILED,
+                detail={"phase": "erase", "error": _err(exc)}, bump_attempts=True,
+            )
+            return
+        state = ObligationState.ERASED if proof.erased else ObligationState.FAILED
+        self._ledger.mark(
+            conn, op, owner_name, ObligationKind.ERASE, state,
+            checksum=proof.checksum,
+            detail={"phase": "erase", **dict(proof.detail)},
+            bump_attempts=True,
+        )
+
+
+def _err(exc: BaseException) -> str:
+    return f"{type(exc).__name__}: {exc}"[:500]
+
+
+__all__ = ["MemoryTransactionService"]

package/src/superlocalmemory/core/worker_pool.py

@@ -84,10 +84,12 @@ class WorkerPool:
         so the worker-side engine resolves the configured default — shared
         memory is opt-in.
         """
-        msg = {
+        msg: dict = {
             "cmd": "recall", "query": query, "limit": limit,
-            "session_id": session_id or "", "fast": bool(fast),
+            "session_id": session_id or "",
         }
+        if fast is not None:
+            msg["fast"] = bool(fast)
         if include_global is not None:
             msg["include_global"] = bool(include_global)
         if include_shared is not None:
@@ -232,12 +234,38 @@ class WorkerPool:
     def _readline_with_timeout(stream, timeout_seconds: float) -> str:
         """Read one line from *stream* with a timeout.
 
-        Uses a daemon thread so the call never blocks the main thread
-        indefinitely. This is the cross-platform replacement for
-        ``selectors`` which fails on Windows pipes.
+        Prefer a deadline-driven selector poll of the stream's file descriptor
+        (POSIX pipes). That path never spawns a helper thread, so a hung
+        worker cannot leak reader threads or pin the pipe FD across timeouts.
+        A thread fallback remains only for streams without a usable fileno
+        (unit-test mocks) and for Windows, where selectors cannot wait on
+        pipes.
 
         Returns the line read, or ``""`` on timeout / error.
         """
+        import selectors
+
+        timeout_seconds = max(0.0, float(timeout_seconds))
+        fd: int | None
+        try:
+            raw_fd = stream.fileno()
+            fd = raw_fd if isinstance(raw_fd, int) else None
+        except (AttributeError, OSError, ValueError, TypeError):
+            fd = None
+
+        # Windows select()/selectors only accept sockets, not subprocess pipes.
+        if fd is not None and sys.platform != "win32":
+            try:
+                with selectors.DefaultSelector() as sel:
+                    sel.register(fd, selectors.EVENT_READ)
+                    events = sel.select(timeout=timeout_seconds)
+                if not events:
+                    return ""
+                line = stream.readline()
+                return line if line else ""
+            except (OSError, ValueError):
+                return ""
+
         result_container: list[str] = []
         error_container: list[Exception] = []
 

package/src/superlocalmemory/encoding/cognitive_consolidator.py

@@ -716,46 +716,88 @@ class CognitiveConsolidator:
 
         Source facts are SOFT-ARCHIVED (HR-04), never deleted.
         Gist embedding stored at float32 (HR-05).
+
+        Ordering guarantee: the gist replacement is committed as a live
+        atomic fact BEFORE source facts are archived.  This ensures there
+        is never a window where neither the sources nor the replacement
+        are recallable.
         """
         block_id = _new_id()
 
-        # Generate gist embedding (full float32 precision — HR-05)
+        # Generate gist embedding BEFORE the transaction.  Embedding generation
+        # is a pure CPU/ML computation with no DB side effects; running it outside
+        # the transaction avoids holding a write lock during a potentially slow
+        # model forward pass.  If generation fails, we abort before any write.
+        # Note: gist_embedding_rowid is currently always None because vec0
+        # insertion requires VectorStore integration (future work).  When that
+        # integration lands, the rowid write must move INSIDE the transaction
+        # below so a rollback removes the embedding row together with the block.
         gist_embedding_rowid: int | None = None
         if self._embedder is not None:
             try:
                 self._embedder.encode(gist.gist_text)
-                # Note: storing in vec0 requires VectorStore integration.
-                # For now, the embedding is generated and the block records
-                # that an embedder was available.
             except Exception as exc:
                 logger.warning("Gist embedding generation failed: %s", exc)
 
-        # Store the consolidated block
-        self._db.store_ccq_block(
-            block_id=block_id,
-            profile_id=profile_id,
-            content=gist.gist_text,
-            source_fact_ids=json.dumps(list(cluster.fact_ids)),
-            gist_embedding_rowid=gist_embedding_rowid,
-            char_count=len(gist.gist_text),
-            cluster_id=cluster.cluster_id,
-        )
-
-        # Archive source facts (HR-04: soft-archive, never delete) through the
-        # lifecycle invariant writer.
+        from superlocalmemory.storage.models import AtomicFact, MemoryRecord
         from superlocalmemory.core.lifecycle_state import set_fact_lifecycle_zone
-        set_fact_lifecycle_zone(
-            self._db, cluster.fact_ids, "archive", profile_id=profile_id,
-        )
-        for fact_id in cluster.fact_ids:
-            # Log access event
-            self._db.execute(
-                "INSERT INTO fact_access_log "
-                "(log_id, fact_id, profile_id, accessed_at, "
-                " access_type, session_id) "
-                "VALUES (?, ?, ?, datetime('now'), 'consolidation', 'ccq')",
-                (_new_id(), fact_id, profile_id),
+
+        # All four writes are wrapped in a single transaction so a mid-write
+        # failure (e.g. store_ccq_block raising) cannot leave an orphan gist
+        # AtomicFact with no corresponding ccq_consolidated_blocks row.
+        # set_fact_lifecycle_zone is transaction-aware and reuses the active
+        # connection rather than opening a nested one.
+        with self._db.transaction():
+            # --- Store replacement fact FIRST (ordering guarantee) ---
+            # The gist is persisted as a live atomic fact so it is immediately
+            # recallable.  Source archival happens only after this write lands.
+            self._db.store_memory(
+                MemoryRecord(
+                    memory_id=block_id,
+                    profile_id=profile_id,
+                    content=gist.gist_text,
+                    session_id="ccq",
+                )
             )
+            self._db.store_fact(
+                AtomicFact(
+                    memory_id=block_id,
+                    profile_id=profile_id,
+                    content=gist.gist_text,
+                    canonical_entities=list(gist.key_entities),
+                    importance=0.8,
+                    confidence=1.0,
+                    session_id="ccq",
+                )
+            )
+
+            # Store the consolidated block — must follow the gist fact INSERT so
+            # the step-1 identify filter (NOT IN ccq_consolidated_blocks) sees the
+            # correct state on any immediate re-run.
+            self._db.store_ccq_block(
+                block_id=block_id,
+                profile_id=profile_id,
+                content=gist.gist_text,
+                source_fact_ids=json.dumps(list(cluster.fact_ids)),
+                gist_embedding_rowid=gist_embedding_rowid,
+                char_count=len(gist.gist_text),
+                cluster_id=cluster.cluster_id,
+            )
+
+            # Archive source facts AFTER replacement is committed (HR-04: soft-archive, never delete)
+            set_fact_lifecycle_zone(
+                self._db, cluster.fact_ids, "archive", profile_id=profile_id,
+            )
+            for fact_id in cluster.fact_ids:
+                # Log access event
+                self._db.execute(
+                    "INSERT INTO fact_access_log "
+                    "(log_id, fact_id, profile_id, accessed_at, "
+                    " access_type, session_id) "
+                    "VALUES (?, ?, ?, datetime('now'), 'consolidation', 'ccq')",
+                    (_new_id(), fact_id, profile_id),
+                )
+
         return block_id
 
     # ------------------------------------------------------------------

package/src/superlocalmemory/encoding/emotional.py

@@ -16,28 +16,89 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
-import math
+import os
+import threading
 from dataclasses import dataclass
 
 logger = logging.getLogger(__name__)
 
-_vader_analyzer = None
+# ---------------------------------------------------------------------------
+# Thread-safe, fork-safe VADER singleton
+#
+# Design notes:
+#   • _vader_analyzer = None means "not yet initialized for this process" OR
+#     "initialized but vaderSentiment is unavailable" (both cases return None
+#     from _get_vader, which causes tag_emotion to fall back to the keyword
+#     heuristic — the correct, observable behavior in both cases).
+#
+#   • _vader_pid tracks which OS process last ran initialization. On fork()
+#     (e.g., under pytest-xdist, multiprocessing, or uwsgi) the child inherits
+#     the parent's _vader_analyzer pointer but the native SentimentIntensityAnalyzer
+#     object may reference memory that is no longer safe in the child's address
+#     space. Re-initializing per-PID guarantees each process gets a fresh, safe
+#     object constructed in its own address space.
+#
+#   • _vader_lock (threading.Lock) prevents the double-initialization race in a
+#     multi-threaded caller (e.g., the ingestion thread-pool in store_pipeline.py
+#     where tag_emotion is called from multiple MaterializerWorker threads
+#     concurrently). Without the lock, Thread A and Thread B can both see
+#     _vader_analyzer is None, both enter the import block, and both try to load
+#     vaderSentiment's lexicon simultaneously — causing memory corruption when
+#     the native C-extension in the same process's numpy/BLAS layer is already
+#     running in a third thread (the LanceDB tokio background thread). The lock
+#     serializes VADER initialization so only one thread loads the lexicon.
+#
+#   • After initialization, reads bypass the lock entirely (fast-path at top of
+#     _get_vader). The GIL guarantees atomic reads of _vader_pid and
+#     _vader_analyzer in CPython; writing _vader_pid AFTER _vader_analyzer
+#     (always, inside the lock) ensures a reader that sees a matching _vader_pid
+#     is guaranteed to also see the correctly initialized _vader_analyzer.
+# ---------------------------------------------------------------------------
+
+_vader_analyzer = None          # None = uninitialized OR unavailable
+_vader_pid: int | None = None   # PID in which _vader_analyzer was initialized
+_vader_lock = threading.Lock()  # Serializes initialization only; not scoring
 
 
 def _get_vader():
-    """Lazy-load VADER to avoid import cost on startup."""
-    global _vader_analyzer
-    if _vader_analyzer is not None:
+    """Lazy-load VADER — thread-safe via lock, fork-safe via PID tracking.
+
+    Returns a SentimentIntensityAnalyzer instance, or None if vaderSentiment
+    is not installed.  Callers must treat None as "fallback to keyword heuristic."
+    """
+    global _vader_analyzer, _vader_pid
+    current_pid = os.getpid()
+
+    # Fast-path: already initialized in this process — no lock needed.
+    # We read _vader_pid first; if it matches current_pid, _vader_analyzer was
+    # written before _vader_pid was set (see ordering in slow-path below),
+    # so the value we read is fully initialized.
+    if _vader_pid == current_pid:
         return _vader_analyzer
-    try:
-        import warnings
-        with warnings.catch_warnings():
-            warnings.filterwarnings("ignore", category=DeprecationWarning, module="vaderSentiment")
-            from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
-            _vader_analyzer = SentimentIntensityAnalyzer()
-    except ImportError:
-        logger.warning("vaderSentiment not installed — emotional tagging disabled")
-        _vader_analyzer = None
+
+    # Slow-path: either first call in this process, or a forked child.
+    with _vader_lock:
+        # Double-checked locking: another thread may have raced us here.
+        if _vader_pid == current_pid:
+            return _vader_analyzer
+
+        # Initialize (or re-initialize) for this process.
+        try:
+            import warnings
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore", category=DeprecationWarning, module="vaderSentiment",
+                )
+                from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
+                _vader_analyzer = SentimentIntensityAnalyzer()
+        except ImportError:
+            logger.warning("vaderSentiment not installed — emotional tagging disabled")
+            _vader_analyzer = None
+
+        # Write _vader_pid LAST so that the fast-path can safely use it as a
+        # "initialization complete" signal.
+        _vader_pid = current_pid
+
     return _vader_analyzer