superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/core/fact_consolidator.py

@@ -32,6 +32,10 @@ import sqlite3
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger("superlocalmemory.fact_consolidator")
 
@@ -41,7 +45,7 @@ _MAX_CONSOLIDATED_CHARS = 2000
 
 
 def consolidate_facts(
-    db_path: str | Path,
+    db_or_path: "Union[DatabaseManager, str, Path]",
     profile_id: str = "default",
     max_clusters: int = 20,
     dry_run: bool = False,
@@ -49,6 +53,20 @@ def consolidate_facts(
 ) -> dict:
     """Find and consolidate clusters of related facts.
 
+    Concurrency fix (v3.8.4 — implements TODO from Fix-A comment):
+    When a DatabaseManager is provided the consolidation now uses per-cluster
+    short write transactions instead of one long raw_connection() that held
+    the write lock across Ollama / Cloud-LLM calls for every cluster after
+    the first.  The fix:
+
+      1. Discover clusters with a short memory_read() (no write lock).
+      2. For each cluster: fetch fact content (memory_read()), generate the
+         summary OUTSIDE any lock (Ollama / Cloud LLM may take 30 s), then
+         do the SAVEPOINT write inside a short memory_write() block.
+
+    SAVEPOINT atomicity per cluster is preserved: _consolidate_cluster still
+    uses SAVEPOINT / RELEASE / ROLLBACK TO internally.
+
     Mode behavior:
       - Mode A: Extractive only (no LLM). Always available.
       - Mode B: Ollama LLM summarization. Falls back to extractive if Ollama down.
@@ -56,7 +74,10 @@ def consolidate_facts(
 
     Returns stats: consolidated, clusters_found, facts_archived, errors.
     """
-    stats = {
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.memory_write import memory_read, memory_write
+
+    stats: dict = {
         "clusters_found": 0,
         "consolidated": 0,
         "facts_archived": 0,
@@ -71,7 +92,79 @@ def consolidate_facts(
             mode_str = getattr(mode, 'value', str(mode)).lower()
             stats["mode"] = mode_str
 
-    conn = sqlite3.connect(str(db_path))
+    if isinstance(db_or_path, DatabaseManager):
+        db_path = db_or_path.db_path
+        try:
+            # Step 1: discover clusters — short read, no write lock.
+            with memory_read(db_path) as rconn:
+                rconn.row_factory = sqlite3.Row
+                clusters = _find_consolidation_clusters(rconn, profile_id, max_clusters)
+            stats["clusters_found"] = len(clusters)
+
+            for entity_id, entity_name, fact_ids in clusters:
+                try:
+                    # Step 2: load fact content for this cluster (read, no write lock).
+                    placeholders = ",".join("?" * len(fact_ids))
+                    with memory_read(db_path) as rconn:
+                        rconn.row_factory = sqlite3.Row
+                        facts = rconn.execute(
+                            f"SELECT fact_id, content, confidence, created_at, "
+                            f"canonical_entities_json, scope, shared_with "
+                            f"FROM atomic_facts "
+                            f"WHERE fact_id IN ({placeholders}) ORDER BY created_at",
+                            fact_ids,
+                        ).fetchall()
+                        facts = [dict(f) for f in facts]
+
+                    if len(facts) < _MIN_CLUSTER_SIZE:
+                        continue
+
+                    # Step 3: generate summary OUTSIDE any write lock.
+                    # Ollama (Mode B) or Cloud LLM (Mode C) may take 30 s here.
+                    summary = _generate_summary(entity_name, facts, config)
+                    if not summary:
+                        continue
+
+                    if dry_run:
+                        stats["consolidated"] += 1
+                        stats["facts_archived"] += len(fact_ids)
+                        continue
+
+                    # Step 4: short per-cluster write — hold lock only for SQL.
+                    with memory_write(db_path) as conn:
+                        conn.row_factory = sqlite3.Row
+                        result = _consolidate_cluster(
+                            conn, profile_id, entity_id, entity_name,
+                            fact_ids, dry_run=False, config=None,
+                            _presummary=summary,
+                        )
+                    if result:
+                        stats["consolidated"] += 1
+                        stats["facts_archived"] += len(fact_ids)
+                except Exception as exc:
+                    logger.warning(
+                        "Consolidation failed for %s: %s",
+                        entity_name, exc, exc_info=True,
+                    )
+                    stats["errors"] += 1
+
+            if stats["consolidated"] > 0:
+                logger.info(
+                    "Fact consolidation: %d clusters merged, %d facts archived",
+                    stats["consolidated"], stats["facts_archived"],
+                )
+        except Exception as exc:
+            logger.error("Fact consolidation failed: %s", exc, exc_info=True)
+            stats["errors"] += 1
+            stats["error_detail"] = str(exc)
+        return stats
+
+    # Backward-compat: str | Path — open own connection.
+    logger.warning(
+        "consolidate_facts: passing a db_path is deprecated — pass a "
+        "DatabaseManager instead (Fix A backward-compat shim active)"
+    )
+    conn = sqlite3.connect(str(db_or_path))
     wal_mode = conn.execute("PRAGMA journal_mode=WAL").fetchone()
     if wal_mode and wal_mode[0] != "wal":
         logger.warning("WAL mode not active, got: %s", wal_mode[0])
@@ -79,33 +172,9 @@ def consolidate_facts(
     conn.row_factory = sqlite3.Row
 
     try:
-        clusters = _find_consolidation_clusters(conn, profile_id, max_clusters)
-        stats["clusters_found"] = len(clusters)
-
-        for entity_id, entity_name, fact_ids in clusters:
-            try:
-                result = _consolidate_cluster(
-                    conn, profile_id, entity_id, entity_name,
-                    fact_ids, dry_run, config,
-                )
-                if result:
-                    stats["consolidated"] += 1
-                    stats["facts_archived"] += len(fact_ids)
-            except Exception as exc:
-                logger.warning(
-                    "Consolidation failed for %s: %s",
-                    entity_name, exc, exc_info=True,
-                )
-                stats["errors"] += 1
-
+        _run_consolidation(conn, profile_id, max_clusters, dry_run, config, stats)
         if not dry_run:
             conn.commit()
-
-        if stats["consolidated"] > 0:
-            logger.info(
-                "Fact consolidation: %d clusters merged, %d facts archived",
-                stats["consolidated"], stats["facts_archived"],
-            )
     except Exception as exc:
         logger.error("Fact consolidation failed: %s", exc, exc_info=True)
         stats["errors"] += 1
@@ -116,6 +185,41 @@ def consolidate_facts(
     return stats
 
 
+def _run_consolidation(
+    conn: sqlite3.Connection,
+    profile_id: str,
+    max_clusters: int,
+    dry_run: bool,
+    config: object | None,
+    stats: dict,
+) -> None:
+    """Core consolidation logic — connection-agnostic inner function."""
+    clusters = _find_consolidation_clusters(conn, profile_id, max_clusters)
+    stats["clusters_found"] = len(clusters)
+
+    for entity_id, entity_name, fact_ids in clusters:
+        try:
+            result = _consolidate_cluster(
+                conn, profile_id, entity_id, entity_name,
+                fact_ids, dry_run, config,
+            )
+            if result:
+                stats["consolidated"] += 1
+                stats["facts_archived"] += len(fact_ids)
+        except Exception as exc:
+            logger.warning(
+                "Consolidation failed for %s: %s",
+                entity_name, exc, exc_info=True,
+            )
+            stats["errors"] += 1
+
+    if stats["consolidated"] > 0:
+        logger.info(
+            "Fact consolidation: %d clusters merged, %d facts archived",
+            stats["consolidated"], stats["facts_archived"],
+        )
+
+
 def _find_consolidation_clusters(
     conn: sqlite3.Connection,
     profile_id: str,
@@ -179,15 +283,25 @@ def _consolidate_cluster(
     fact_ids: list[str],
     dry_run: bool,
     config: object | None = None,
+    *,
+    _presummary: str | None = None,
 ) -> dict | None:
     """Merge a cluster of facts into one consolidated fact.
 
     All writes are wrapped in a SAVEPOINT for atomicity — if any step fails,
     the entire cluster consolidation is rolled back.
+
+    _presummary (v3.8.4): when the caller supplies a pre-computed summary
+    (generated OUTSIDE the write lock), skip the _generate_summary() call.
+    This is the short-lock path used by the DatabaseManager branch of
+    consolidate_facts().  The str | Path backward-compat path still calls
+    _generate_summary() inline (legacy behaviour, no regression).
     """
     c = conn.cursor()
 
-    # Load fact contents including canonical_entities_json
+    # Load fact contents including canonical_entities_json.
+    # Even when _presummary is provided we still re-read from the DB inside
+    # the write lock so the SAVEPOINT has a fresh, authoritative facts list.
     placeholders = ",".join("?" * len(fact_ids))
     facts = c.execute(
         f"SELECT fact_id, content, confidence, created_at, canonical_entities_json, "
@@ -200,7 +314,11 @@ def _consolidate_cluster(
     if len(facts) < _MIN_CLUSTER_SIZE:
         return None
 
-    summary = _generate_summary(entity_name, facts, config)
+    if _presummary is not None:
+        summary = _presummary
+    else:
+        # Legacy path (str | Path caller) — may call Ollama with write lock held.
+        summary = _generate_summary(entity_name, facts, config)
     if not summary:
         return None