superlocalmemory 3.8.0 → 3.8.2

package/src/superlocalmemory/core/store_pipeline.py

@@ -11,8 +11,10 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 
 from __future__ import annotations
 
-import logging
+import hashlib
 import json
+import logging
+import uuid
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -21,7 +23,9 @@ if TYPE_CHECKING:
     from superlocalmemory.storage.database import DatabaseManager
 
 from superlocalmemory.storage.models import (
-    AtomicFact, FactType, MemoryRecord,
+    AtomicFact,
+    FactType,
+    MemoryRecord,
 )
 
 logger = logging.getLogger(__name__)
@@ -30,6 +34,105 @@ logger = logging.getLogger(__name__)
 _INIT_LANGEVIN_RADIUS = 0.05
 
 
+def _ingestion_effect_id(operation_id: str, *parts: object) -> str:
+    """Return a stable ID for a relational effect owned by one ingestion."""
+    if not operation_id:
+        return uuid.uuid4().hex
+    payload = "\0".join((operation_id, *(str(part) for part in parts)))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
+
+
+def _record_fact_entity_association(
+    db: DatabaseManager,
+    *,
+    operation_id: str,
+    profile_id: str,
+    fact_id: str,
+    entity_id: str,
+) -> None:
+    """Apply one fact/entity count effect in O(1), exactly once."""
+    if not operation_id:
+        db.increment_entity_fact_count(entity_id, profile_id)
+        return
+    with db.transaction():
+        claimed = db.execute(
+            "INSERT INTO fact_entity_associations "
+            "(profile_id,fact_id,entity_id,first_operation_id,count_applied) "
+            "SELECT ?,?,?,?,"
+            "CASE WHEN fact.rowid > repair.target_fact_rowid THEN 1 ELSE 0 END "
+            "FROM canonical_entities AS entity "
+            "JOIN atomic_facts AS fact "
+            "ON fact.fact_id=? AND fact.profile_id=? "
+            "JOIN fact_entity_association_repair_state AS repair "
+            "ON repair.repair_key='historical-backfill' "
+            "WHERE entity.entity_id=? AND entity.profile_id=? "
+            "ON CONFLICT(profile_id,fact_id,entity_id) DO UPDATE SET "
+            "count_applied=excluded.count_applied,"
+            "first_operation_id=excluded.first_operation_id "
+            "WHERE fact_entity_associations.count_applied=0 "
+            "AND excluded.count_applied=1 "
+            "RETURNING count_applied",
+            (
+                profile_id,
+                fact_id,
+                entity_id,
+                operation_id,
+                fact_id,
+                profile_id,
+                entity_id,
+                profile_id,
+            ),
+        )
+        if claimed and int(claimed[0]["count_applied"]) == 1:
+            db.execute(
+                "UPDATE canonical_entities SET fact_count=fact_count+1 "
+                "WHERE entity_id=? AND profile_id=?",
+                (entity_id, profile_id),
+            )
+        elif not claimed:
+            # An empty result has TWO causes: (a) the 'historical-backfill'
+            # repair-state row is missing — M028 DDL ran but its INSERT rolled
+            # back (partial migration) — so the JOIN returned zero rows and the
+            # main INSERT produced no output; or (b) the association already
+            # exists and the ON CONFLICT update was correctly skipped (an
+            # idempotent re-run). Only (a) needs the fallback insert; (b) is a
+            # no-op. Disambiguate by probing the repair-state table directly —
+            # a DIFFERENT table, so this stays a constant-query, idempotent
+            # effect and never re-fires the insert on an already-applied row.
+            repair_ready = db.execute(
+                "SELECT 1 FROM fact_entity_association_repair_state "
+                "WHERE repair_key='historical-backfill' LIMIT 1",
+            )
+            if repair_ready:
+                # (b) idempotent skip — the association was already applied.
+                return
+            # (a) partial migration: insert with count_applied=1 (no historical-
+            # rowid check needed — all associations in this state are
+            # post-migration). ON CONFLICT DO NOTHING makes this retry-safe.
+            fallback = db.execute(
+                "INSERT INTO fact_entity_associations "
+                "(profile_id,fact_id,entity_id,first_operation_id,count_applied) "
+                "SELECT ?,?,?,?,1 "
+                "FROM canonical_entities AS entity "
+                "JOIN atomic_facts AS fact "
+                "ON fact.fact_id=? AND fact.profile_id=? "
+                "WHERE entity.entity_id=? AND entity.profile_id=? "
+                "ON CONFLICT(profile_id,fact_id,entity_id) DO NOTHING "
+                "RETURNING count_applied",
+                (
+                    profile_id, fact_id, entity_id, operation_id,
+                    fact_id, profile_id,
+                    entity_id, profile_id,
+                ),
+            )
+            if fallback:
+                db.execute(
+                    "UPDATE canonical_entities SET fact_count=fact_count+1 "
+                    "WHERE entity_id=? AND profile_id=?",
+                    (entity_id, profile_id),
+                )
+
+
 def _init_langevin_position(dim: int = 8) -> list[float]:
     """Initialize Langevin position near origin for a new fact.
 
@@ -60,7 +163,7 @@ def enrich_fact(
     temporal_parser: Any,
 ) -> AtomicFact:
     """Enrich fact with embeddings, entities, temporal, emotional data."""
-    from superlocalmemory.encoding.emotional import tag_emotion, emotional_importance_boost
+    from superlocalmemory.encoding.emotional import emotional_importance_boost, tag_emotion
     from superlocalmemory.encoding.signal_inference import infer_signal
 
     embedding = embedder.embed(fact.content) if embedder else None
@@ -189,6 +292,9 @@ def run_store(
     ingestion_source_type: str = "store",
     ingestion_operation_id: str = "",
     derivation_report: dict[str, bool] | None = None,
+    precompleted_derivation_stages: frozenset[str] = frozenset(),
+    materialization_progress: dict[str, Any] | None = None,
+    materialization_checkpoint: Any = None,
 ) -> list[str]:
     """Store content and extract structured facts. Returns fact_ids.
 
@@ -286,10 +392,19 @@ def run_store(
         )
         db.store_memory(record)
 
-    extraction_complete = False
-    consolidation_complete = consolidator is not None
-    canonicalization_complete = entity_resolver is not None
-    graph_complete = graph_builder is not None
+    extraction_complete = "extraction" in precompleted_derivation_stages
+    consolidation_complete = (
+        "consolidation" in precompleted_derivation_stages
+        or consolidator is not None
+    )
+    canonicalization_complete = (
+        "canonicalization" in precompleted_derivation_stages
+        or entity_resolver is not None
+    )
+    graph_complete = (
+        "graph" in precompleted_derivation_stages
+        or graph_builder is not None
+    )
     temporal_complete = True
     provenance_complete = provenance is not None
 
@@ -330,7 +445,6 @@ def run_store(
             and content.strip()
             and len(content.strip()) >= 40
             and len(content.strip().split()) >= _MIN_VERBATIM_WORDS):
-        import uuid
         import re as _re
         _verbatim_text = content.strip()
         # Extract entities using the same regex as fact_extractor
@@ -374,7 +488,6 @@ def run_store(
     # their data should NEVER be silently dropped. The min-length and min-word filters
     # are designed for automatic conversation extraction, not explicit user storage.
     if not facts and content.strip():
-        import uuid
         facts = [AtomicFact(
             fact_id=uuid.uuid4().hex[:16],
             content=content.strip(),
@@ -403,6 +516,22 @@ def run_store(
             temporal_parser=temporal_parser,
         )
 
+        if (
+            materialization_progress is not None
+            and not materialization_progress.get("relational_started", False)
+        ):
+            if materialization_checkpoint is not None:
+                materialization_checkpoint(
+                    "relational_started",
+                    (),
+                    {
+                        "pipeline_started": True,
+                        "relational_started": True,
+                        "pipeline": False,
+                    },
+                )
+            materialization_progress["relational_started"] = True
+
         is_queryable_promotion = fact.fact_id in queryable_ids
         if is_queryable_promotion:
             db.update_fact(fact.fact_id, {
@@ -515,6 +644,8 @@ def run_store(
 
         if fact.fact_id not in stored_ids:
             stored_ids.append(fact.fact_id)
+            if materialization_progress is not None:
+                materialization_progress["fact_ids"] = tuple(stored_ids)
 
         # Dual-write embedding to ANN index + vector store (embed on-demand if
         # a consolidated ADD fact arrived without one). See _upsert_fact_vectors.
@@ -611,12 +742,17 @@ def run_store(
             for eid in fact.canonical_entities:
                 observation_builder.update_profile(eid, fact, profile_id)
 
-        # Increment fact_count for each linked canonical entity (scoped to profile).
+        # The normalized association key makes this O(1) and exactly-once
+        # across retries, crashes, and separate operations that consolidate to
+        # the same fact.
         for eid in fact.canonical_entities:
-            try:
-                db.increment_entity_fact_count(eid, profile_id)
-            except Exception:
-                pass  # Non-critical — entity may have been deleted
+            _record_fact_entity_association(
+                db,
+                operation_id=ingestion_operation_id,
+                profile_id=profile_id,
+                fact_id=fact.fact_id,
+                entity_id=eid,
+            )
         if scene_builder:
             scene_builder.assign_to_scene(fact, profile_id)
 
@@ -627,6 +763,13 @@ def run_store(
             from superlocalmemory.storage.models import TemporalEvent
             for eid in fact.canonical_entities:
                 event = TemporalEvent(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        eid,
+                        "observed",
+                    ),
                     profile_id=profile_id, entity_id=eid,
                     fact_id=fact.fact_id,
                     scope=fact.scope,
@@ -644,8 +787,16 @@ def run_store(
             from superlocalmemory.encoding.foresight import extract_foresight_signals
             from superlocalmemory.storage.models import TemporalEvent as _TE
             foresight_signals = extract_foresight_signals(fact)
-            for sig in foresight_signals:
+            for signal_index, sig in enumerate(foresight_signals):
                 f_event = _TE(
+                    event_id=_ingestion_effect_id(
+                        ingestion_operation_id,
+                        "temporal",
+                        fact.fact_id,
+                        sig.get("entity_id", ""),
+                        "foresight",
+                        signal_index,
+                    ),
                     profile_id=profile_id,
                     entity_id=sig.get("entity_id", ""),
                     fact_id=fact.fact_id,
@@ -679,11 +830,6 @@ def run_store(
 
     logger.info("Stored %d facts (session=%s)", len(stored_ids), session_id)
 
-    # Post-operation hooks (audit, trust signal, event bus)
-    hook_ctx["fact_ids"] = stored_ids
-    hook_ctx["fact_count"] = len(stored_ids)
-    hooks.run_post("store", hook_ctx)
-
     if derivation_report is not None:
         derivation_report.update({
             "extraction": extraction_complete,
@@ -694,6 +840,35 @@ def run_store(
             "provenance": provenance_complete,
         })
 
+    # Stage observations and emitted fact IDs are recorded before optional
+    # post-hooks.  A hook failure must not erase the durable retry boundary and
+    # cause extraction/consolidation to run again.
+    if materialization_progress is not None:
+        materialization_progress["fact_ids"] = tuple(stored_ids)
+        materialization_progress["relational_complete"] = True
+        materialization_progress["post_hooks"] = False
+    if materialization_checkpoint is not None and stored_ids:
+        materialization_checkpoint(
+            "relational_complete",
+            tuple(stored_ids),
+            {
+                "pipeline_started": True,
+                "relational_started": True,
+                "pipeline": True,
+                "post_hooks": False,
+                "relational": True,
+                **dict(derivation_report or {}),
+            },
+        )
+
+    # Post-operation hooks (audit, trust signal, event bus)
+    hook_ctx["ingestion_operation_id"] = ingestion_operation_id
+    hook_ctx["fact_ids"] = stored_ids
+    hook_ctx["fact_count"] = len(stored_ids)
+    hooks.run_post("store", hook_ctx)
+    if materialization_progress is not None:
+        materialization_progress["post_hooks"] = True
+
     # Phase 5: Step-count trigger for lightweight consolidation (L7)
     if consolidation_engine is not None:
         try:

package/src/superlocalmemory/hooks/hook_handlers.py

@@ -26,6 +26,8 @@ import urllib.error
 import urllib.request
 from pathlib import Path
 
+from superlocalmemory import __version__
+
 # ---------------------------------------------------------------------------
 # Cross-platform temp paths
 # ---------------------------------------------------------------------------
@@ -221,7 +223,10 @@ def _codex_mcp_session_init(project_dir: str, payload: dict) -> dict:
         proc.stdin.write(json.dumps({
             "jsonrpc": "2.0", "id": 1, "method": "initialize",
             "params": {"protocolVersion": "2024-11-05", "capabilities": {},
-                       "clientInfo": {"name": "superlocalmemory-codex-hook", "version": "3.7"}},
+                       "clientInfo": {
+                           "name": "superlocalmemory-codex-hook",
+                           "version": __version__,
+                       }},
         }) + "\n")
         proc.stdin.flush()
         proc.stdout.readline()

package/src/superlocalmemory/hooks/portable_kit.py

@@ -240,11 +240,12 @@ def connect_ide(
     here: bool = False,
     profile: str | None = None,
     agents_md_source: Callable[[], str] | None = None,
+    dry_run: bool = False,
 ) -> dict[str, Any]:
     """Wire SLM into the target IDE config via merge-not-clobber.
 
     Returns a result dict:
-        {ide, mcp_config: wrote|merged|unchanged|skipped|error,
+        {ide, mcp_config: wrote|merged|unchanged|would_write|skipped|error,
          mcp_path, agents_md: wrote|skipped(...)|unchanged|error,
          servers_preserved: int, error: str|None}
     """
@@ -265,6 +266,13 @@ def connect_ide(
         )
         return result
 
+    if ide_id == "vscode-copilot" and not here:
+        result["error"] = (
+            "VS Code / Copilot integration is project-scoped. "
+            "Run `slm connect vscode-copilot --here` from the project root."
+        )
+        return result
+
     # Step 1a — claude-code short-circuit (AC6)
     if desc.fmt == "":
         print(CLAUDE_CODE_PLUGIN_POINTER)
@@ -325,7 +333,28 @@ def connect_ide(
         result["servers_preserved"] = max(0, pre_count - (0 if pre_slm is None else 1))
         result["mcp_config"] = mcp_status
 
-    # Step 6 — atomic write
+    # Step 6 — atomic write. A dry run deliberately exercises the merge and
+    # packaged-asset lookup path without touching a user-owned IDE config.
+    if dry_run:
+        result["mcp_config"] = "would_write"
+        if desc.agents_md_path is None:
+            result["agents_md"] = "skipped(unsupported)"
+        elif agents_md_source is None:
+            result["agents_md"] = "skipped(no-source)"
+        else:
+            try:
+                source_content = agents_md_source()
+            except Exception as exc:
+                result["agents_md"] = "error(source-unavailable)"
+                result["error"] = f"AGENTS.md asset lookup failed: {exc}"
+                return result
+            if not isinstance(source_content, str) or not source_content.strip():
+                result["agents_md"] = "error(empty-source)"
+                result["error"] = "AGENTS.md asset is empty"
+                return result
+            result["agents_md"] = "would_write"
+        return result
+
     try:
         _atomic_write(config_path, data, desc.fmt)
     except Exception as exc:
@@ -528,6 +557,9 @@ def _handle_agents_md(
     except Exception as exc:
         logger.warning("agents_md_source() failed: %s — skipping AGENTS.md write", exc)
         return "skipped(source-error)"
+    if not isinstance(source_content, str) or not source_content.strip():
+        logger.warning("agents_md_source() returned no content — skipping AGENTS.md write")
+        return "skipped(no-source-content)"
 
     # Read existing content
     existing = ""

package/src/superlocalmemory/learning/model_rollback.py

@@ -34,6 +34,8 @@ import sqlite3
 from datetime import datetime, timedelta, timezone
 from typing import Final
 
+from superlocalmemory.learning.model_cache import invalidate as invalidate_model_cache
+
 logger = logging.getLogger(__name__)
 
 
@@ -235,6 +237,7 @@ class ModelRollback:
                 )
                 return False
 
+        invalidate_model_cache(self._profile_id)
         logger.warning(
             "AUTO-ROLLBACK profile=%s reason=%s observations=%d "
             "baseline_ndcg=%.4f current_ndcg=%.4f",

package/src/superlocalmemory/learning/ranker_retrain_online.py

@@ -33,6 +33,7 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Final
 
+from superlocalmemory.learning.model_cache import invalidate as invalidate_model_cache
 from superlocalmemory.learning.ranker_common import (
     _build_training_matrix,
     _compute_eval_metrics,
@@ -389,6 +390,7 @@ def _promote_candidate(
                 (json.dumps(meta), candidate_id),
             )
             conn.commit()
+            invalidate_model_cache(profile_id)
             return True
         except sqlite3.Error as exc:
             conn.rollback()

package/src/superlocalmemory/learning/reward.py

@@ -73,6 +73,24 @@ _DWELL_MAX_MS: Final[int] = 3_600_000  # 1 h
 _BUSY_TIMEOUT_MS: Final[int] = 50
 
 
+def _feed_source_quality(
+    memory_db_path: Path,
+    rewards: list[tuple[str, str, list[str], float]],
+) -> None:
+    """Fail-soft provenance bridge kept outside the reward transaction."""
+    try:
+        from superlocalmemory.learning.source_quality import (
+            update_source_quality_for_reward_batch,
+        )
+        update_source_quality_for_reward_batch(
+            memory_db_path=memory_db_path,
+            learning_db_path=memory_db_path.parent / "learning.db",
+            rewards=rewards,
+        )
+    except Exception as exc:  # noqa: BLE001 - reward finalization must survive
+        logger.debug("source-quality reward feed failed (non-fatal): %s", exc)
+
+
 # ---------------------------------------------------------------------------
 # Signal contract — names match the manifest A.1 label formula.
 # ---------------------------------------------------------------------------
@@ -604,6 +622,7 @@ class EngagementRewardModel:
                 # lock-scope — SQLite rows are tied to the connection.
                 _pid = pending["profile_id"]
                 _qid = pending["recall_query_id"]
+                _facts_json = str(pending["fact_ids_json"] or "[]")
         except sqlite3.Error as exc:
             logger.debug("finalize_outcome SQLite error: %s", exc)
             return _FALLBACK_REWARD
@@ -633,6 +652,21 @@ class EngagementRewardModel:
             except Exception as exc:  # noqa: BLE001 — defence in depth
                 logger.debug("feed_recall_settled failed (non-fatal): %s", exc)
 
+        try:
+            _fact_ids = json.loads(_facts_json)
+            if not isinstance(_fact_ids, list):
+                _fact_ids = []
+        except (TypeError, ValueError, json.JSONDecodeError):
+            _fact_ids = []
+        _feed_source_quality(
+            self._db,
+            [(
+                str(_pid),
+                str(outcome_id),
+                [str(fid) for fid in _fact_ids if fid],
+                float(reward),
+            )],
+        )
         return reward
 
     # ------------------------------------------------------------------
@@ -749,6 +783,22 @@ class EngagementRewardModel:
                     except sqlite3.Error:
                         conn.execute("ROLLBACK")
                         raise
+                reward_sources = []
+                for item in i_chunk:
+                    try:
+                        fact_ids = json.loads(str(item[2] or "[]"))
+                    except (TypeError, ValueError, json.JSONDecodeError):
+                        fact_ids = []
+                    reward_sources.append((
+                        str(item[1]),
+                        str(item[0]),
+                        (
+                            [str(fid) for fid in fact_ids if fid]
+                            if isinstance(fact_ids, list) else []
+                        ),
+                        float(item[4]),
+                    ))
+                _feed_source_quality(self._db, reward_sources)
         except sqlite3.Error as exc:  # pragma: no cover — defensive
             logger.debug("reap_stale SQLite error: %s", exc)
             return written