knowledge-rag 4.9.0__tar.gz → 4.9.2__tar.gz

{knowledge_rag-4.9.0 → knowledge_rag-4.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: knowledge-rag
-Version: 4.9.0
+Version: 4.9.2
 Summary: Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers.
 Project-URL: Homepage, https://github.com/lyonzin/knowledge-rag
 Project-URL: Repository, https://github.com/lyonzin/knowledge-rag
@@ -636,6 +636,8 @@ Native MCP
 
 **Complete client configuration guide with JSON schemas per client:** [docs/INSTALLATION.md#use-with-other-mcp-clients →](docs/INSTALLATION.md#use-with-other-mcp-clients)
 
+> ⚠️ **Operating model — read once.** Your MCP client (Claude Desktop, Code, Cursor, etc.) **spawns and owns the knowledge-rag server process**. Do **NOT** also run `knowledge-rag` in a terminal while your MCP client is open — two writer processes against the same `data_dir` corrupt the ChromaDB HNSW segment on Windows (see [issue #216](https://github.com/lyonzin/knowledge-rag/issues/216)). Recommended baseline: add `"KNOWLEDGE_RAG_SINGLE_INSTANCE": "1"` to the `env` block of your MCP config as a safety net. Full explanation + how to run CLI commands safely: [docs/single-instance.md](docs/single-instance.md).
+
 ---
 
 ## ⚙️ Configuration in 30 seconds

{knowledge_rag-4.9.0 → knowledge_rag-4.9.2}/README.md

@@ -588,6 +588,8 @@ Native MCP
 
 **Complete client configuration guide with JSON schemas per client:** [docs/INSTALLATION.md#use-with-other-mcp-clients →](docs/INSTALLATION.md#use-with-other-mcp-clients)
 
+> ⚠️ **Operating model — read once.** Your MCP client (Claude Desktop, Code, Cursor, etc.) **spawns and owns the knowledge-rag server process**. Do **NOT** also run `knowledge-rag` in a terminal while your MCP client is open — two writer processes against the same `data_dir` corrupt the ChromaDB HNSW segment on Windows (see [issue #216](https://github.com/lyonzin/knowledge-rag/issues/216)). Recommended baseline: add `"KNOWLEDGE_RAG_SINGLE_INSTANCE": "1"` to the `env` block of your MCP config as a safety net. Full explanation + how to run CLI commands safely: [docs/single-instance.md](docs/single-instance.md).
+
 ---
 
 ## ⚙️ Configuration in 30 seconds

{knowledge_rag-4.9.0 → knowledge_rag-4.9.2}/mcp_server/__init__.py

@@ -8,7 +8,7 @@ import sys  # noqa: I001
 _original_stdout = sys.stdout
 sys.stdout = sys.stderr
 
-__version__ = "4.9.0"
+__version__ = "4.9.2"
 __author__ = "Ailton Rocha (Lyon.)"
 
 from .config import Config  # noqa: E402

knowledge_rag-4.9.2/mcp_server/preflight.py

@@ -0,0 +1,171 @@
+"""Startup preflight checks for persistent ChromaDB state.
+
+GH #216 (v4.9.1): probe retries + inconclusive-on-timeout + atomic rename.
+The previous single-attempt quarantine turned any transient failure
+(file locks after client restart, slow disk, cold GPU init) into a
+full-corpus reindex loop on Windows stdio MCP clients. The reporter's
+forensic analysis on issue #216 documented the failure mode and the
+fixes below were validated on their environment before merging.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+from .config import BASE_DIR, config
+
+PROBE_ATTEMPTS = 3
+PROBE_RETRY_DELAY_SECONDS = 5
+PROBE_TIMEOUT_SECONDS = 30
+
+
+def _log(msg: str) -> None:
+    """Emit preflight message to stderr and append to data/preflight.log.
+
+    Persisting to a file matters because stdio MCP clients hide the server's
+    stderr in their own logs — without the file, users could only ever see
+    the truncated exit code, not the actual Chroma stack trace.
+    """
+    line = f"[{datetime.now().isoformat(timespec='seconds')}] {msg}"
+    print(line, file=sys.stderr)
+    try:
+        log_file = config.data_dir / "preflight.log"
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        with log_file.open("a", encoding="utf-8") as f:
+            f.write(line + "\n")
+    except OSError:
+        pass
+
+
+def _backup_active_index(reason: str) -> Path:
+    """Atomically move active ChromaDB state aside so the server can rebuild.
+
+    Uses os.rename (atomic on same volume) instead of shutil.move — the
+    fallback copy+rmtree path in shutil.move leaks a partial copy in
+    backups/ AND a partially deleted live chroma_db/ when another process
+    holds a segment file open (WinError 32).
+
+    The two moves (metadata JSON + chroma_db dir) must be all-or-nothing.
+    If we moved chroma_db first and metadata failed (antivirus scanning
+    the JSON, brief file lock), restart would find `chroma_db` gone but
+    `index_metadata.json` still live — incremental reindex would then
+    read the orphan metadata, mark every doc as "already indexed", and
+    silently produce an empty collection. To prevent that, metadata is
+    moved first (small file, low lock probability) and chroma_db is
+    wrapped in try/except that rolls the metadata rename back before
+    re-raising. Restart then sees the pristine pre-quarantine state
+    and re-runs preflight cleanly.
+    """
+    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    backup_dir = config.data_dir / "backups" / f"auto-repair-{stamp}"
+    backup_dir.mkdir(parents=True, exist_ok=False)
+
+    metadata_file = config.data_dir / "index_metadata.json"
+    metadata_backup = backup_dir / f"index_metadata.{reason}.json"
+    metadata_moved = False
+    if metadata_file.exists():
+        os.rename(str(metadata_file), str(metadata_backup))
+        metadata_moved = True
+
+    if config.chroma_dir.exists():
+        try:
+            os.rename(str(config.chroma_dir), str(backup_dir / f"chroma_db.{reason}"))
+        except OSError:
+            if metadata_moved:
+                # Rollback the metadata move so the two files stay consistent.
+                # If the rollback itself fails (unlikely — dst dir is fresh)
+                # we still re-raise the original OSError so the caller can
+                # exit cleanly instead of leaving the system in a wedged state.
+                try:
+                    os.rename(str(metadata_backup), str(metadata_file))
+                except OSError:
+                    pass
+            raise
+
+    return backup_dir
+
+
+def _probe_chroma(timeout_seconds: int = PROBE_TIMEOUT_SECONDS) -> subprocess.CompletedProcess[str]:
+    """Check Chroma in a child process so native crashes do not kill MCP startup."""
+    code = r"""
+import chromadb
+
+from mcp_server.config import config
+
+if not config.chroma_dir.exists():
+    print("missing")
+    raise SystemExit(0)
+
+client = chromadb.PersistentClient(path=str(config.chroma_dir))
+collection = client.get_or_create_collection(name=config.collection_name)
+print(collection.count())
+"""
+    env = os.environ.copy()
+    env.setdefault("KNOWLEDGE_RAG_DIR", str(BASE_DIR))
+    return subprocess.run(
+        [sys.executable, "-c", code],
+        cwd=str(BASE_DIR),
+        env=env,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        timeout=timeout_seconds,
+        check=False,
+    )
+
+
+def run_preflight(timeout_seconds: int = PROBE_TIMEOUT_SECONDS) -> bool:
+    """Return True when active Chroma state was moved aside for repair.
+
+    Runs the probe up to PROBE_ATTEMPTS times so transient failures
+    (file locks lingering from a previous PID, slow disk, cold GPU init)
+    do not trigger a destructive quarantine. TimeoutExpired is treated
+    as inconclusive — the index is kept and the user can debug from
+    preflight.log without losing the corpus. Only PROBE_ATTEMPTS
+    consecutive non-zero exits quarantine.
+
+    If os.rename fails while quarantining (e.g. another process still
+    holds a segment file open on Windows) the index is left untouched
+    and the process exits with EX_TEMPFAIL (75) instead of crashing
+    halfway with a partial move.
+    """
+    last_result: subprocess.CompletedProcess[str] | None = None
+    for attempt in range(1, PROBE_ATTEMPTS + 1):
+        try:
+            result = _probe_chroma(timeout_seconds=timeout_seconds)
+        except subprocess.TimeoutExpired:
+            _log(
+                f"[PREFLIGHT] attempt {attempt}/{PROBE_ATTEMPTS}: probe timed out "
+                f"after {timeout_seconds}s — inconclusive, keeping index"
+            )
+            return False
+        if result.returncode == 0:
+            if attempt > 1:
+                _log(f"[PREFLIGHT] attempt {attempt}: OK (recovered from transient failure)")
+            return False
+        stderr_tail = (result.stderr or "").strip()[-1500:]
+        _log(
+            f"[PREFLIGHT] attempt {attempt}/{PROBE_ATTEMPTS}: probe failed with code {result.returncode}\n{stderr_tail}"
+        )
+        last_result = result
+        if attempt < PROBE_ATTEMPTS:
+            time.sleep(PROBE_RETRY_DELAY_SECONDS)
+
+    assert last_result is not None
+    reason = "segfault" if last_result.returncode in (-11, 139) else "failed"
+    _log(f"[RECOVERY] preflight failed {PROBE_ATTEMPTS}x (last exit code {last_result.returncode}); moving index aside")
+    try:
+        backup_dir = _backup_active_index(reason)
+    except OSError as exc:
+        _log(
+            f"[PREFLIGHT] could not move index aside ({exc}); leaving it untouched "
+            f"to avoid partial corruption. Retry after freeing file handles."
+        )
+        raise SystemExit(75) from exc
+    _log(f"[RECOVERY] moved active index to {backup_dir}")
+    return True

{knowledge_rag-4.9.0 → knowledge_rag-4.9.2}/mcp_server/server.py

@@ -4793,9 +4793,15 @@ def main():
                     print("[FALLBACK] Attempting regular index instead...")
                     stats = orchestrator.index_all(force=True)
             elif orchestrator.collection.count() == 0:
-                print("[INFO] No documents indexed. Running initial indexing...")
-                stats = orchestrator.index_all()
-                print(f"[INFO] Indexed {stats['indexed']} documents with {stats['chunks_added']} chunks")
+                # GH #216: initial indexing runs in a background thread so it
+                # does not block the MCP `initialize` handshake. Blocking here
+                # for a large corpus (~150k chunks takes minutes) caused stdio
+                # clients to timeout, kill the process, and restart into an
+                # empty index — an endless full-reindex loop. Users poll
+                # get_reindex_status() to track progress.
+                print("[INFO] No documents indexed. Starting initial indexing in background...")
+                orchestrator.start_reindex_background("incremental")
+                print("[INFO] Initial indexing runs in background. Track progress with get_reindex_status.")
 
             # Start file watcher for auto-reindex on document changes
             if os.environ.get("KNOWLEDGE_RAG_WATCHER_DISABLED", "").strip() == "1":

{knowledge_rag-4.9.0 → knowledge_rag-4.9.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "knowledge-rag"
-version = "4.9.0"
+version = "4.9.2"
 description = "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers."
 readme = "README.md"
 license = {text = "MIT"}

knowledge_rag-4.9.0/mcp_server/preflight.py

@@ -1,74 +0,0 @@
-"""Startup preflight checks for persistent ChromaDB state."""
-
-from __future__ import annotations
-
-import os
-import shutil
-import subprocess
-import sys
-from datetime import datetime
-from pathlib import Path
-
-from .config import BASE_DIR, config
-
-
-def _backup_active_index(reason: str) -> Path:
-    """Move active ChromaDB state aside so the server can rebuild cleanly."""
-    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
-    backup_dir = config.data_dir / "backups" / f"auto-repair-{stamp}"
-    backup_dir.mkdir(parents=True, exist_ok=False)
-
-    if config.chroma_dir.exists():
-        shutil.move(str(config.chroma_dir), str(backup_dir / f"chroma_db.{reason}"))
-
-    metadata_file = config.data_dir / "index_metadata.json"
-    if metadata_file.exists():
-        shutil.move(str(metadata_file), str(backup_dir / f"index_metadata.{reason}.json"))
-
-    return backup_dir
-
-
-def _probe_chroma(timeout_seconds: int = 30) -> subprocess.CompletedProcess[str]:
-    """Check Chroma in a child process so native crashes do not kill MCP startup."""
-    code = r"""
-import chromadb
-
-from mcp_server.config import config
-
-if not config.chroma_dir.exists():
-    print("missing")
-    raise SystemExit(0)
-
-client = chromadb.PersistentClient(path=str(config.chroma_dir))
-collection = client.get_or_create_collection(name=config.collection_name)
-print(collection.count())
-"""
-    env = os.environ.copy()
-    env.setdefault("KNOWLEDGE_RAG_DIR", str(BASE_DIR))
-    return subprocess.run(
-        [sys.executable, "-c", code],
-        cwd=str(BASE_DIR),
-        env=env,
-        text=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        timeout=timeout_seconds,
-        check=False,
-    )
-
-
-def run_preflight(timeout_seconds: int = 30) -> bool:
-    """Return True when active Chroma state was moved aside for repair."""
-    result = _probe_chroma(timeout_seconds=timeout_seconds)
-    if result.returncode == 0:
-        return False
-
-    reason = "segfault" if result.returncode in (-11, 139) else "failed"
-    backup_dir = _backup_active_index(reason)
-    print(
-        f"[RECOVERY] Chroma preflight failed with code {result.returncode}; moved active index to {backup_dir}",
-        file=sys.stderr,
-    )
-    if result.stderr:
-        print(result.stderr[-2000:], file=sys.stderr)
-    return True