knowledge-rag 4.2.0__tar.gz → 4.3.1__tar.gz

knowledge_rag-4.2.0/README.md → knowledge_rag-4.3.1/PKG-INFO

@@ -1,3 +1,51 @@
+Metadata-Version: 2.4
+Name: knowledge-rag
+Version: 4.3.1
+Summary: Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers.
+Project-URL: Homepage, https://github.com/lyonzin/knowledge-rag
+Project-URL: Repository, https://github.com/lyonzin/knowledge-rag
+Project-URL: Issues, https://github.com/lyonzin/knowledge-rag/issues
+Project-URL: Changelog, https://github.com/lyonzin/knowledge-rag/releases
+Author-email: "Lyon." <lyonzin@users.noreply.github.com>
+License: MIT
+License-File: LICENSE
+Keywords: bm25,chromadb,claude-code,embeddings,fastembed,hybrid-search,knowledge-base,local-ai,mcp,rag,reranking,retrieval-augmented-generation,semantic-search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.11
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: chromadb>=1.4.0
+Requires-Dist: fastembed[reranking]>=0.4.0
+Requires-Dist: mcp>=1.6.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: openpyxl>=3.1.0
+Requires-Dist: pymupdf>=1.23.0
+Requires-Dist: python-docx>=1.0.0
+Requires-Dist: python-pptx>=1.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.33.0
+Requires-Dist: watchdog>=4.0.0
+Provides-Extra: gpu
+Requires-Dist: nvidia-cublas-cu12; extra == 'gpu'
+Requires-Dist: nvidia-cuda-runtime-cu12; extra == 'gpu'
+Requires-Dist: nvidia-cudnn-cu12; extra == 'gpu'
+Requires-Dist: nvidia-cufft-cu12; extra == 'gpu'
+Requires-Dist: nvidia-curand-cu12; extra == 'gpu'
+Requires-Dist: nvidia-cusolver-cu12; extra == 'gpu'
+Requires-Dist: nvidia-cusparse-cu12; extra == 'gpu'
+Requires-Dist: nvidia-nvjitlink-cu12; extra == 'gpu'
+Requires-Dist: onnxruntime-gpu>=1.14.0; extra == 'gpu'
+Provides-Extra: server
+Requires-Dist: uvicorn>=0.20.0; extra == 'server'
+Description-Content-Type: text/markdown
+
 # Knowledge RAG
 
 <div align="center">
@@ -17,7 +65,7 @@
 ### Your docs, your machine, zero cloud. Claude Code searches them natively.
 
 Drop your PDFs, markdown, code, notebooks — **1800+ files, 39K chunks, indexed in under 3 minutes.**<br/>
-Hybrid search (BM25 + semantic vectors + cross-encoder reranking) through 12 MCP tools.<br/>
+Hybrid search (BM25 + semantic vectors + cross-encoder reranking) through 13 MCP tools.<br/>
 Everything runs locally via ONNX. No Docker, no Ollama, no API keys, no data leaves your machine.
 
 ```
@@ -26,7 +74,7 @@ pip install knowledge-rag → restart Claude Code → search_knowledge("your que
 
 ---
 
-**12 MCP Tools** | **Hybrid Search + Reranking** | **20 File Formats** | **Optional NVIDIA GPU** | **100% Local**
+**13 MCP Tools** | **Hybrid Search + Reranking** | **20 File Formats** | **Optional NVIDIA GPU** | **100% Local**
 
 [What's New](#whats-new-in-v420) | [Supported Formats](#supported-formats) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
 
@@ -81,7 +129,7 @@ Or via CLI: `knowledge-rag --transport sse`
 - **Prometheus metrics**: `/metrics` endpoint on separate port
 - **Bearer auth**: Token validation for SSE/HTTP connections
 
-All 12 MCP tools are instrumented with `@rate_limited` and `@instrument` decorators — zero overhead when features are disabled. Default transport remains **stdio** for full backwards compatibility.
+All 13 MCP tools are instrumented with `@rate_limited` and `@instrument` decorators — zero overhead when features are disabled. Default transport remains **stdio** for full backwards compatibility.
 
 > **Migration**: Existing users need zero changes. SSE mode is opt-in via `server.transport: "sse"` in config.yaml. See [Configuration](#configuration) for details.
 
@@ -205,7 +253,7 @@ See [Changelog](#changelog) for full history.
 | **MMR Diversification** | Maximal Marginal Relevance reduces redundant results |
 | **Persistent Model Cache** | Embedding models cached in `models_cache/` — survives reboots |
 | **Auto-Migration** | Detects embedding dimension mismatch and rebuilds automatically |
-| **12 MCP Tools** | Full CRUD + search + evaluation via Claude Code |
+| **13 MCP Tools** | Full CRUD + search + evaluation via Claude Code |
 
 ---
 
@@ -217,7 +265,7 @@ See [Changelog](#changelog) for full history.
 flowchart TB
     subgraph MCP["MCP SERVER (FastMCP)"]
         direction TB
-        TOOLS["12 MCP Tools<br/>search | get | add | update | remove<br/>reindex | list | stats | url | similar | evaluate"]
+        TOOLS["13 MCP Tools<br/>search | get | add | update | remove<br/>reindex | reindex_status | list | stats | url | similar | evaluate"]
     end
 
     subgraph SEARCH["HYBRID SEARCH ENGINE"]
@@ -392,7 +440,53 @@ flowchart LR
 - Claude Code CLI
 - *…or any other MCP client (Claude Desktop, Cursor, VS Code, Antigravity, opencode, Windsurf) — see [Use with other MCP clients](#use-with-other-mcp-clients)*
 - ~200MB disk for model cache (auto-downloaded on first run)
-- *Optional:* NVIDIA GPU + CUDA for accelerated embeddings (`pip install knowledge-rag[gpu]` + `models.embedding.gpu: true` in config)
+- *Optional:* NVIDIA GPU + CUDA 12 for accelerated embeddings (see [GPU Acceleration](#gpu-acceleration) below)
+
+### GPU Acceleration
+
+GPU mode accelerates embedding generation during indexing and search. It requires an NVIDIA GPU with CUDA 12 support. No GPU? No problem — the server runs on CPU by default and GPU is entirely optional.
+
+**Requirements:**
+
+| Component | Minimum | How to check / get it |
+|-----------|---------|----------------------|
+| NVIDIA GPU (Turing+) | RTX 20xx / 30xx / 40xx / 50xx, or Tesla T4+ | `nvidia-smi` |
+| NVIDIA Driver | ≥ 525 | `nvidia-smi` — [nvidia.com/drivers](https://www.nvidia.com/drivers) |
+| CUDA 12 runtime | Provided by pip packages below | Automatic |
+
+**Setup (2 steps):**
+
+```bash
+# 1. Install GPU dependencies (onnxruntime-gpu + all CUDA 12 runtime DLLs)
+pip install knowledge-rag[gpu]
+
+# 2. Enable in config.yaml
+# models:
+#   embedding:
+#     gpu: true
+```
+
+The `[gpu]` extra installs `onnxruntime-gpu` plus 7 NVIDIA CUDA 12 packages (`cublas`, `cudnn`, `cuda-runtime`, `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`) so you don't need a full CUDA Toolkit install.
+
+**Verify GPU is active:**
+
+On server startup, look for the GPU status banner:
+```
+============================================================
+  GPU STATUS: ACTIVE
+  Provider:   CUDAExecutionProvider
+  Device:     NVIDIA GeForce RTX 3080 Ti
+  VRAM:       12.0 GB
+============================================================
+```
+
+Or programmatically:
+```bash
+python -c "import onnxruntime; print(onnxruntime.get_available_providers())"
+# Should include: 'CUDAExecutionProvider'
+```
+
+> **Fallback**: If CUDA is unavailable at runtime (wrong driver, missing DLLs, no GPU), the server falls back to CPU automatically with a `[WARN]` log — it never crashes. The `gpu: true` config is a preference, not a requirement.
 
 ### Install Methods
 
@@ -663,7 +757,7 @@ search_knowledge("lateral movement strategies", hybrid_alpha=1.0)
 
 ### Indexing
 
-Documents are automatically indexed on first startup. To manage the index:
+Documents are automatically indexed on first startup. All reindex operations run **in background** — they return immediately and you poll progress via `get_reindex_status()`:
 
 ```python
 # Incremental: only re-index changed files (fast)
@@ -674,6 +768,10 @@ reindex_documents(force=True)
 
 # Nuclear rebuild: delete everything, re-embed all (use after model change)
 reindex_documents(full_rebuild=True)
+
+# Poll progress (lightweight, no full stats computation)
+get_reindex_status()
+# → {"reindex": {"active": true, "percent": 56, "progress": "2090/3734", ...}}
 ```
 
 ### Evaluating Retrieval Quality
@@ -755,14 +853,39 @@ Retrieve the full content of a specific document.
 
 #### `reindex_documents`
 
-Index or reindex all documents in the knowledge base.
+Index or reindex all documents in the knowledge base. **Runs in background** — returns immediately. Poll progress via `get_reindex_status()`.
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `force` | bool | false | Smart reindex: detects changes, rebuilds BM25. Fast. |
 | `full_rebuild` | bool | false | Nuclear rebuild: deletes everything, re-embeds all documents. Use after model change. |
 
-**Returns:** JSON with indexing statistics (indexed, updated, skipped, deleted, chunks_added, chunks_removed, dedup_skipped, elapsed_seconds).
+**Returns:** `{"status": "started", "operation": "..."}` immediately. If already running, returns `{"status": "already_running", "progress": "1200/3734"}`.
+
+---
+
+#### `get_reindex_status`
+
+Get the current status of a background reindex operation. Lightweight — does not compute full index statistics.
+
+**Returns (active):**
+```json
+{
+  "status": "success",
+  "reindex": {
+    "active": true,
+    "operation": "nuclear_rebuild",
+    "progress": "1200/3734",
+    "percent": 32,
+    "indexed": 1200,
+    "skipped": 0,
+    "errors": 0,
+    "started_at": "2026-06-17T18:29:49"
+  }
+}
+```
+
+**Returns (idle):** `{"status": "success", "reindex": {"active": false}}`
 
 ---
 
@@ -1071,7 +1194,7 @@ For `.md` files, chunking splits at `##` and `###` header boundaries first. Sect
 |-------|---------|-------------|
 | `models.embedding.model` | `BAAI/bge-small-en-v1.5` | Embedding model (ONNX, runs locally) |
 | `models.embedding.dimensions` | 384 | Vector dimensions (must match model) |
-| `models.embedding.gpu` | false | Enable CUDA GPU acceleration. Requires `pip install knowledge-rag[gpu]` |
+| `models.embedding.gpu` | false | Enable CUDA GPU acceleration. See [GPU Acceleration](#gpu-acceleration) for full setup |
 | `models.reranker.enabled` | true | Enable cross-encoder reranking |
 | `models.reranker.model` | `Xenova/ms-marco-MiniLM-L-6-v2` | Reranker model |
 | `models.reranker.top_k_multiplier` | 3 | Fetch N*multiplier candidates for reranking |
@@ -1324,6 +1447,64 @@ Common issues:
 
 ## Changelog
 
+### Unreleased
+
+### v4.3.1 (2026-06-22) — Hybrid Search Fixes
+
+- **FIX**: Accept `"general"` as a valid category in `search_knowledge`. The parser hardcodes `"general"` as the fallback in `_detect_category` (`ingestion.py`), but the validator only built `valid_categories` from `config.keyword_routes` + `config.category_mappings.values()` — so users who customized `config.yaml` and dropped the default `"general": "general"` mapping hit `Invalid category` even though the index contained `general` documents. Validator now always tolerates `"general"`. (#98, thanks @Hohlas)
+- **FIX**: Skip BM25-only search results when Chroma can no longer resolve the chunk ID. Stale BM25 indices (typically right after `remove_document` or in the window between async reindex and BM25 rebuild) returned hits whose `collection.get()` came back empty; the previous fallback inserted entries with `document=""` / `metadata={}` into the reranker, polluting results with empty matches. The pipeline now `continue`s past those, dropping the stale hit cleanly. (#98, thanks @Hohlas)
+- **TEST**: Added `tests/test_pr98_regression.py` (4 tests) pinning both contracts so future refactors cannot silently revert either fix. Test count baseline: 227 → 231. (#99)
+- **CI**: Bumped `[tool.mypy] python_version` from 3.11 to 3.12 to accept PEP 695 `type` statements in the numpy stub (`numpy/__init__.pyi`) which were breaking the Pillar 7 strict gate. Only affects static analysis; `requires-python = ">=3.11"` unchanged. (#100)
+
+### v4.3.0 (2026-06-17) — Async Reindex, GPU CUDA 12, 13th MCP Tool
+
+- **NEW**: `get_reindex_status` MCP tool — lightweight reindex progress polling without computing full index stats. Returns active/idle status, percent, processed/total, errors, and last result.
+- **NEW**: `reindex_documents` now runs in background via daemon thread — returns immediately with `{"status": "started"}`. Eliminates MCP timeout on large document sets (5K+ files). Concurrent calls return `already_running` with current progress.
+- **NEW**: GPU acceleration with full CUDA 12 support — `onnxruntime-gpu` + 7 NVIDIA pip packages (`cublas`, `cudnn`, `cuda-runtime`, `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`). Server auto-detects GPU on startup with 4-step verification (providers, DLLs, nvidia-smi, session creation). Falls back to CPU gracefully.
+- **NEW**: `_setup_cuda_dll_paths()` adds NVIDIA pip package DLL directories to `PATH` automatically on Windows — onnxruntime finds CUDA 12 DLLs without a full CUDA Toolkit install.
+- **DEPS**: `[gpu]` extra expanded from 3 to 8 packages (added `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`).
+- **FIX**: GPU status reporting now uses actual ONNX session creation test instead of just checking `get_available_providers()` — prevents false "GPU ACTIVE" when CUDA DLLs are missing.
+- **DOCS**: GPU Acceleration section rewritten with complete requirements table, setup steps, verification instructions, and fallback behavior.
+- **DOCS**: Tool reference updated — `reindex_documents` async behavior documented, `get_reindex_status` reference added.
+- **TEST**: Backwards-compat baseline updated for 13 MCP tools.
+
+### v4.2.0 (2026-06-17) — Search Performance & Output Quality
+
+- **PERF**: Custom inverted-index BM25 replaces `rank-bm25` full-corpus scan — 128× faster keyword search on 50K+ chunk corpora. Only documents containing query terms are scored via posting lists.
+- **PERF**: `numpy.argpartition` for O(n) top-k selection instead of O(n log n) sort.
+- **PERF**: Batched adjacent chunk fetch — single ChromaDB `collection.get()` call replaces N round-trips per result.
+- **PERF**: O(1) reverse lookup via `_source_to_docid` dict eliminates linear scans of `_indexed_docs` in `search_similar`, `update_document`, `remove_document`, and `_expand_with_adjacent_chunks`.
+- **NEW**: `snippet_mode` parameter on `search_knowledge` (default: `true`) — truncates content to ~500 chars at natural break points with `content_length` field. Reduces token consumption by ~72%.
+- **NEW**: `min_score` parameter on `search_knowledge` (default: `0.0`) — filters results below a normalized relevance threshold. Response includes `filtered_by_score` count.
+- **NEW**: `filtered_by_score` field in search response JSON for transparency.
+- **DEPS**: `numpy` added as direct dependency (was transitive via fastembed); `rank-bm25` import removed from server.py.
+- **TEST**: 6 new tests for `min_score` filtering and `snippet_mode` truncation.
+- **TEST**: Updated backwards-compat baseline to include new `search_knowledge` parameters.
+
+### v4.1.2 (2026-06-17)
+
+- **FIX**: `_save_metadata` dict snapshot prevents concurrent modification crash during file watcher events.
+- **STYLE**: ruff format applied to server.py.
+
+### v4.1.1 (2026-06-17)
+
+- **FIX**: All `_indexed_docs` iterations now use `list()` snapshot, preventing `dictionary changed size during iteration` crash when FileWatcher modifies the index concurrently with MCP tool calls (affects `search_knowledge`, `search_similar`, `update_document`, `remove_document`, `evaluate_retrieval`, `list_categories`, `list_documents`)
+
+### v4.1.0 (2026-06-17)
+
+- **Added:** `query_expansion_groups` config for symmetric synonym expansion (#92)
+- **Improved:** `expand_query()` now returns deterministic expansion order (set → ordered list with dedup)
+
+### v4.0.1 (2026-06-16)
+
+- **FIX**: Orphan cleanup now runs before indexing loop, preventing chunk loss when files are moved (#90).
+- **FIX**: Chunk deduplication is now per-document instead of global, preventing cross-document chunk deletion (#91).
+- **FIX**: Added `on_moved` handler to `DocumentWatcher` for proper file move detection.
+- **FIX**: Startup preflight probes ChromaDB in a child process and moves crashing persistent indexes to `data/backups/auto-repair-*` before MCP initialization.
+- **FIX**: Reranker load failures now fall back to RRF ordering instead of failing `search_knowledge` on offline machines.
+- **FIX**: Virtualenv project-root detection now handles Python symlinks that resolve to the system interpreter.
+- **NEW**: `knowledge-rag-guarded` console script kept as an explicit guarded startup alias.
+
 ### v4.0.0 (2026-06-09) — Enterprise Concurrent Access
 
 - **NEW**: SSE and streamable-http transport modes — 1 server serves N clients (`server.transport: "sse"` in config.yaml or `--transport sse` CLI).
@@ -1331,7 +1512,7 @@ Common issues:
 - **NEW**: ChromaDB WAL mode enabled automatically in SSE/HTTP mode for concurrent read performance.
 - **NEW**: Optional rate limiting — sliding-window counter, configurable RPM and burst, disabled by default.
 - **NEW**: Optional Prometheus metrics endpoint — tool call counts, latency histograms, separate port, disabled by default.
-- **NEW**: All 12 MCP tools instrumented with `@rate_limited` and `@instrument` decorators (zero-cost when disabled).
+- **NEW**: All 13 MCP tools instrumented with `@rate_limited` and `@instrument` decorators (zero-cost when disabled).
 - **NEW**: `--transport` CLI override for Docker/systemd deployments.
 - **NEW**: `pip install knowledge-rag[server]` optional dependency for SSE/HTTP (uvicorn).
 - **CHANGED**: SSE/HTTP mode auto-enables single-instance lock (port collision prevention).
@@ -1359,7 +1540,7 @@ Common issues:
 - **NEW** Property-based fuzzing of all parsers via Hypothesis (`tests/test_ingestion_property.py`) — 200 random examples per CI run.
 - **NEW** Memory baseline regression tests (`tests/test_memory_baseline.py`, cross-platform via psutil) — RSS bounded under 1000 queries; nightly soak amplifies to 50K iterations.
 - **NEW** Property/locale/format/preset matrices (`tests/test_presets.py`, `tests/test_locale.py`, `tests/test_format_smoke.py`).
-- **NEW** Backwards-compatibility regression tests (`tests/test_backwards_compat.py`) — legacy YAML configs from v3.6.0 / v3.7.0 still parse; all 12 MCP tool parameter names frozen.
+- **NEW** Backwards-compatibility regression tests (`tests/test_backwards_compat.py`) — legacy YAML configs from v3.6.0 / v3.7.0 still parse; all 13 MCP tool parameter names frozen.
 - **NEW** AST-based public API surface diff (`scripts/check_api_surface.py`) — any breaking change blocks merge, baseline at `.github/api-surface-baseline.json`.
 - **NEW** CHANGELOG enforcement (`scripts/check_changelog.py`) — user-facing PRs must add a bullet under `## Unreleased`; bypass via `skip-changelog` label.
 - **NEW** Test count anti-regression (`scripts/check_test_count.py`) — guards against silent test deletion.
@@ -1389,45 +1570,6 @@ Common issues:
 - **CHORE**: pytest `tmp_path_retention_count=1` to avoid Windows atexit cleanup race in CI.
 - **ROADMAP**: Tracked v4.0 shared-service architecture (one daemon, many thin MCP clients) as the long-term fix for multi-process resource duplication. (#34)
 
-### Unreleased
-
-### v4.2.0 (2026-06-17) — Search Performance & Output Quality
-
-- **PERF**: Custom inverted-index BM25 replaces `rank-bm25` full-corpus scan — 128× faster keyword search on 50K+ chunk corpora. Only documents containing query terms are scored via posting lists.
-- **PERF**: `numpy.argpartition` for O(n) top-k selection instead of O(n log n) sort.
-- **PERF**: Batched adjacent chunk fetch — single ChromaDB `collection.get()` call replaces N round-trips per result.
-- **PERF**: O(1) reverse lookup via `_source_to_docid` dict eliminates linear scans of `_indexed_docs` in `search_similar`, `update_document`, `remove_document`, and `_expand_with_adjacent_chunks`.
-- **NEW**: `snippet_mode` parameter on `search_knowledge` (default: `true`) — truncates content to ~500 chars at natural break points with `content_length` field. Reduces token consumption by ~72%.
-- **NEW**: `min_score` parameter on `search_knowledge` (default: `0.0`) — filters results below a normalized relevance threshold. Response includes `filtered_by_score` count.
-- **NEW**: `filtered_by_score` field in search response JSON for transparency.
-- **DEPS**: `numpy` added as direct dependency (was transitive via fastembed); `rank-bm25` import removed from server.py.
-- **TEST**: 6 new tests for `min_score` filtering and `snippet_mode` truncation.
-- **TEST**: Updated backwards-compat baseline to include new `search_knowledge` parameters.
-
-### v4.1.2 (2026-06-17)
-
-- **FIX**: `_save_metadata` dict snapshot prevents concurrent modification crash during file watcher events.
-- **STYLE**: ruff format applied to server.py.
-
-### v4.1.1 (2026-06-17)
-
-- **FIX**: All `_indexed_docs` iterations now use `list()` snapshot, preventing `dictionary changed size during iteration` crash when FileWatcher modifies the index concurrently with MCP tool calls (affects `search_knowledge`, `search_similar`, `update_document`, `remove_document`, `evaluate_retrieval`, `list_categories`, `list_documents`)
-
-### v4.1.0 (2026-06-17)
-
-- **Added:** `query_expansion_groups` config for symmetric synonym expansion (#92)
-- **Improved:** `expand_query()` now returns deterministic expansion order (set → ordered list with dedup)
-
-### v4.0.1 (2026-06-16)
-
-- **FIX**: Orphan cleanup now runs before indexing loop, preventing chunk loss when files are moved (#90).
-- **FIX**: Chunk deduplication is now per-document instead of global, preventing cross-document chunk deletion (#91).
-- **FIX**: Added `on_moved` handler to `DocumentWatcher` for proper file move detection.
-- **FIX**: Startup preflight probes ChromaDB in a child process and moves crashing persistent indexes to `data/backups/auto-repair-*` before MCP initialization.
-- **FIX**: Reranker load failures now fall back to RRF ordering instead of failing `search_knowledge` on offline machines.
-- **FIX**: Virtualenv project-root detection now handles Python symlinks that resolve to the system interpreter.
-- **NEW**: `knowledge-rag-guarded` console script kept as an explicit guarded startup alias.
-
 ### v3.6.2 (2026-04-23)
 
 - **INFRA**: NPM provenance attestation (SLSA supply chain security), full README on npm page

knowledge_rag-4.2.0/PKG-INFO → knowledge_rag-4.3.1/README.md

@@ -1,43 +1,3 @@
-Metadata-Version: 2.4
-Name: knowledge-rag
-Version: 4.2.0
-Summary: Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 12 MCP Tools + 20 Format Parsers. Zero external servers.
-Project-URL: Homepage, https://github.com/lyonzin/knowledge-rag
-Project-URL: Repository, https://github.com/lyonzin/knowledge-rag
-Project-URL: Issues, https://github.com/lyonzin/knowledge-rag/issues
-Project-URL: Changelog, https://github.com/lyonzin/knowledge-rag/releases
-Author-email: "Lyon." <lyonzin@users.noreply.github.com>
-License: MIT
-License-File: LICENSE
-Keywords: bm25,chromadb,claude-code,embeddings,fastembed,hybrid-search,knowledge-base,local-ai,mcp,rag,reranking,retrieval-augmented-generation,semantic-search
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: Science/Research
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Text Processing :: Indexing
-Requires-Python: >=3.11
-Requires-Dist: beautifulsoup4>=4.12.0
-Requires-Dist: chromadb>=1.4.0
-Requires-Dist: fastembed[reranking]>=0.4.0
-Requires-Dist: mcp>=1.6.0
-Requires-Dist: numpy>=1.24.0
-Requires-Dist: openpyxl>=3.1.0
-Requires-Dist: pymupdf>=1.23.0
-Requires-Dist: python-docx>=1.0.0
-Requires-Dist: python-pptx>=1.0.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: requests>=2.33.0
-Requires-Dist: watchdog>=4.0.0
-Provides-Extra: gpu
-Requires-Dist: onnxruntime-gpu>=1.14.0; extra == 'gpu'
-Provides-Extra: server
-Requires-Dist: uvicorn>=0.20.0; extra == 'server'
-Description-Content-Type: text/markdown
-
 # Knowledge RAG
 
 <div align="center">
@@ -57,7 +17,7 @@ Description-Content-Type: text/markdown
 ### Your docs, your machine, zero cloud. Claude Code searches them natively.
 
 Drop your PDFs, markdown, code, notebooks — **1800+ files, 39K chunks, indexed in under 3 minutes.**<br/>
-Hybrid search (BM25 + semantic vectors + cross-encoder reranking) through 12 MCP tools.<br/>
+Hybrid search (BM25 + semantic vectors + cross-encoder reranking) through 13 MCP tools.<br/>
 Everything runs locally via ONNX. No Docker, no Ollama, no API keys, no data leaves your machine.
 
 ```
@@ -66,7 +26,7 @@ pip install knowledge-rag → restart Claude Code → search_knowledge("your que
 
 ---
 
-**12 MCP Tools** | **Hybrid Search + Reranking** | **20 File Formats** | **Optional NVIDIA GPU** | **100% Local**
+**13 MCP Tools** | **Hybrid Search + Reranking** | **20 File Formats** | **Optional NVIDIA GPU** | **100% Local**
 
 [What's New](#whats-new-in-v420) | [Supported Formats](#supported-formats) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
 
@@ -121,7 +81,7 @@ Or via CLI: `knowledge-rag --transport sse`
 - **Prometheus metrics**: `/metrics` endpoint on separate port
 - **Bearer auth**: Token validation for SSE/HTTP connections
 
-All 12 MCP tools are instrumented with `@rate_limited` and `@instrument` decorators — zero overhead when features are disabled. Default transport remains **stdio** for full backwards compatibility.
+All 13 MCP tools are instrumented with `@rate_limited` and `@instrument` decorators — zero overhead when features are disabled. Default transport remains **stdio** for full backwards compatibility.
 
 > **Migration**: Existing users need zero changes. SSE mode is opt-in via `server.transport: "sse"` in config.yaml. See [Configuration](#configuration) for details.
 
@@ -245,7 +205,7 @@ See [Changelog](#changelog) for full history.
 | **MMR Diversification** | Maximal Marginal Relevance reduces redundant results |
 | **Persistent Model Cache** | Embedding models cached in `models_cache/` — survives reboots |
 | **Auto-Migration** | Detects embedding dimension mismatch and rebuilds automatically |
-| **12 MCP Tools** | Full CRUD + search + evaluation via Claude Code |
+| **13 MCP Tools** | Full CRUD + search + evaluation via Claude Code |
 
 ---
 
@@ -257,7 +217,7 @@ See [Changelog](#changelog) for full history.
 flowchart TB
     subgraph MCP["MCP SERVER (FastMCP)"]
         direction TB
-        TOOLS["12 MCP Tools<br/>search | get | add | update | remove<br/>reindex | list | stats | url | similar | evaluate"]
+        TOOLS["13 MCP Tools<br/>search | get | add | update | remove<br/>reindex | reindex_status | list | stats | url | similar | evaluate"]
     end
 
     subgraph SEARCH["HYBRID SEARCH ENGINE"]
@@ -432,7 +392,53 @@ flowchart LR
 - Claude Code CLI
 - *…or any other MCP client (Claude Desktop, Cursor, VS Code, Antigravity, opencode, Windsurf) — see [Use with other MCP clients](#use-with-other-mcp-clients)*
 - ~200MB disk for model cache (auto-downloaded on first run)
-- *Optional:* NVIDIA GPU + CUDA for accelerated embeddings (`pip install knowledge-rag[gpu]` + `models.embedding.gpu: true` in config)
+- *Optional:* NVIDIA GPU + CUDA 12 for accelerated embeddings (see [GPU Acceleration](#gpu-acceleration) below)
+
+### GPU Acceleration
+
+GPU mode accelerates embedding generation during indexing and search. It requires an NVIDIA GPU with CUDA 12 support. No GPU? No problem — the server runs on CPU by default and GPU is entirely optional.
+
+**Requirements:**
+
+| Component | Minimum | How to check / get it |
+|-----------|---------|----------------------|
+| NVIDIA GPU (Turing+) | RTX 20xx / 30xx / 40xx / 50xx, or Tesla T4+ | `nvidia-smi` |
+| NVIDIA Driver | ≥ 525 | `nvidia-smi` — [nvidia.com/drivers](https://www.nvidia.com/drivers) |
+| CUDA 12 runtime | Provided by pip packages below | Automatic |
+
+**Setup (2 steps):**
+
+```bash
+# 1. Install GPU dependencies (onnxruntime-gpu + all CUDA 12 runtime DLLs)
+pip install knowledge-rag[gpu]
+
+# 2. Enable in config.yaml
+# models:
+#   embedding:
+#     gpu: true
+```
+
+The `[gpu]` extra installs `onnxruntime-gpu` plus 7 NVIDIA CUDA 12 packages (`cublas`, `cudnn`, `cuda-runtime`, `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`) so you don't need a full CUDA Toolkit install.
+
+**Verify GPU is active:**
+
+On server startup, look for the GPU status banner:
+```
+============================================================
+  GPU STATUS: ACTIVE
+  Provider:   CUDAExecutionProvider
+  Device:     NVIDIA GeForce RTX 3080 Ti
+  VRAM:       12.0 GB
+============================================================
+```
+
+Or programmatically:
+```bash
+python -c "import onnxruntime; print(onnxruntime.get_available_providers())"
+# Should include: 'CUDAExecutionProvider'
+```
+
+> **Fallback**: If CUDA is unavailable at runtime (wrong driver, missing DLLs, no GPU), the server falls back to CPU automatically with a `[WARN]` log — it never crashes. The `gpu: true` config is a preference, not a requirement.
 
 ### Install Methods
 
@@ -703,7 +709,7 @@ search_knowledge("lateral movement strategies", hybrid_alpha=1.0)
 
 ### Indexing
 
-Documents are automatically indexed on first startup. To manage the index:
+Documents are automatically indexed on first startup. All reindex operations run **in background** — they return immediately and you poll progress via `get_reindex_status()`:
 
 ```python
 # Incremental: only re-index changed files (fast)
@@ -714,6 +720,10 @@ reindex_documents(force=True)
 
 # Nuclear rebuild: delete everything, re-embed all (use after model change)
 reindex_documents(full_rebuild=True)
+
+# Poll progress (lightweight, no full stats computation)
+get_reindex_status()
+# → {"reindex": {"active": true, "percent": 56, "progress": "2090/3734", ...}}
 ```
 
 ### Evaluating Retrieval Quality
@@ -795,14 +805,39 @@ Retrieve the full content of a specific document.
 
 #### `reindex_documents`
 
-Index or reindex all documents in the knowledge base.
+Index or reindex all documents in the knowledge base. **Runs in background** — returns immediately. Poll progress via `get_reindex_status()`.
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `force` | bool | false | Smart reindex: detects changes, rebuilds BM25. Fast. |
 | `full_rebuild` | bool | false | Nuclear rebuild: deletes everything, re-embeds all documents. Use after model change. |
 
-**Returns:** JSON with indexing statistics (indexed, updated, skipped, deleted, chunks_added, chunks_removed, dedup_skipped, elapsed_seconds).
+**Returns:** `{"status": "started", "operation": "..."}` immediately. If already running, returns `{"status": "already_running", "progress": "1200/3734"}`.
+
+---
+
+#### `get_reindex_status`
+
+Get the current status of a background reindex operation. Lightweight — does not compute full index statistics.
+
+**Returns (active):**
+```json
+{
+  "status": "success",
+  "reindex": {
+    "active": true,
+    "operation": "nuclear_rebuild",
+    "progress": "1200/3734",
+    "percent": 32,
+    "indexed": 1200,
+    "skipped": 0,
+    "errors": 0,
+    "started_at": "2026-06-17T18:29:49"
+  }
+}
+```
+
+**Returns (idle):** `{"status": "success", "reindex": {"active": false}}`
 
 ---
 
@@ -1111,7 +1146,7 @@ For `.md` files, chunking splits at `##` and `###` header boundaries first. Sect
 |-------|---------|-------------|
 | `models.embedding.model` | `BAAI/bge-small-en-v1.5` | Embedding model (ONNX, runs locally) |
 | `models.embedding.dimensions` | 384 | Vector dimensions (must match model) |
-| `models.embedding.gpu` | false | Enable CUDA GPU acceleration. Requires `pip install knowledge-rag[gpu]` |
+| `models.embedding.gpu` | false | Enable CUDA GPU acceleration. See [GPU Acceleration](#gpu-acceleration) for full setup |
 | `models.reranker.enabled` | true | Enable cross-encoder reranking |
 | `models.reranker.model` | `Xenova/ms-marco-MiniLM-L-6-v2` | Reranker model |
 | `models.reranker.top_k_multiplier` | 3 | Fetch N*multiplier candidates for reranking |
@@ -1364,6 +1399,64 @@ Common issues:
 
 ## Changelog
 
+### Unreleased
+
+### v4.3.1 (2026-06-22) — Hybrid Search Fixes
+
+- **FIX**: Accept `"general"` as a valid category in `search_knowledge`. The parser hardcodes `"general"` as the fallback in `_detect_category` (`ingestion.py`), but the validator only built `valid_categories` from `config.keyword_routes` + `config.category_mappings.values()` — so users who customized `config.yaml` and dropped the default `"general": "general"` mapping hit `Invalid category` even though the index contained `general` documents. Validator now always tolerates `"general"`. (#98, thanks @Hohlas)
+- **FIX**: Skip BM25-only search results when Chroma can no longer resolve the chunk ID. Stale BM25 indices (typically right after `remove_document` or in the window between async reindex and BM25 rebuild) returned hits whose `collection.get()` came back empty; the previous fallback inserted entries with `document=""` / `metadata={}` into the reranker, polluting results with empty matches. The pipeline now `continue`s past those, dropping the stale hit cleanly. (#98, thanks @Hohlas)
+- **TEST**: Added `tests/test_pr98_regression.py` (4 tests) pinning both contracts so future refactors cannot silently revert either fix. Test count baseline: 227 → 231. (#99)
+- **CI**: Bumped `[tool.mypy] python_version` from 3.11 to 3.12 to accept PEP 695 `type` statements in the numpy stub (`numpy/__init__.pyi`) which were breaking the Pillar 7 strict gate. Only affects static analysis; `requires-python = ">=3.11"` unchanged. (#100)
+
+### v4.3.0 (2026-06-17) — Async Reindex, GPU CUDA 12, 13th MCP Tool
+
+- **NEW**: `get_reindex_status` MCP tool — lightweight reindex progress polling without computing full index stats. Returns active/idle status, percent, processed/total, errors, and last result.
+- **NEW**: `reindex_documents` now runs in background via daemon thread — returns immediately with `{"status": "started"}`. Eliminates MCP timeout on large document sets (5K+ files). Concurrent calls return `already_running` with current progress.
+- **NEW**: GPU acceleration with full CUDA 12 support — `onnxruntime-gpu` + 7 NVIDIA pip packages (`cublas`, `cudnn`, `cuda-runtime`, `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`). Server auto-detects GPU on startup with 4-step verification (providers, DLLs, nvidia-smi, session creation). Falls back to CPU gracefully.
+- **NEW**: `_setup_cuda_dll_paths()` adds NVIDIA pip package DLL directories to `PATH` automatically on Windows — onnxruntime finds CUDA 12 DLLs without a full CUDA Toolkit install.
+- **DEPS**: `[gpu]` extra expanded from 3 to 8 packages (added `cufft`, `cusparse`, `cusolver`, `curand`, `nvjitlink`).
+- **FIX**: GPU status reporting now uses actual ONNX session creation test instead of just checking `get_available_providers()` — prevents false "GPU ACTIVE" when CUDA DLLs are missing.
+- **DOCS**: GPU Acceleration section rewritten with complete requirements table, setup steps, verification instructions, and fallback behavior.
+- **DOCS**: Tool reference updated — `reindex_documents` async behavior documented, `get_reindex_status` reference added.
+- **TEST**: Backwards-compat baseline updated for 13 MCP tools.
+
+### v4.2.0 (2026-06-17) — Search Performance & Output Quality
+
+- **PERF**: Custom inverted-index BM25 replaces `rank-bm25` full-corpus scan — 128× faster keyword search on 50K+ chunk corpora. Only documents containing query terms are scored via posting lists.
+- **PERF**: `numpy.argpartition` for O(n) top-k selection instead of O(n log n) sort.
+- **PERF**: Batched adjacent chunk fetch — single ChromaDB `collection.get()` call replaces N round-trips per result.
+- **PERF**: O(1) reverse lookup via `_source_to_docid` dict eliminates linear scans of `_indexed_docs` in `search_similar`, `update_document`, `remove_document`, and `_expand_with_adjacent_chunks`.
+- **NEW**: `snippet_mode` parameter on `search_knowledge` (default: `true`) — truncates content to ~500 chars at natural break points with `content_length` field. Reduces token consumption by ~72%.
+- **NEW**: `min_score` parameter on `search_knowledge` (default: `0.0`) — filters results below a normalized relevance threshold. Response includes `filtered_by_score` count.
+- **NEW**: `filtered_by_score` field in search response JSON for transparency.
+- **DEPS**: `numpy` added as direct dependency (was transitive via fastembed); `rank-bm25` import removed from server.py.
+- **TEST**: 6 new tests for `min_score` filtering and `snippet_mode` truncation.
+- **TEST**: Updated backwards-compat baseline to include new `search_knowledge` parameters.
+
+### v4.1.2 (2026-06-17)
+
+- **FIX**: `_save_metadata` dict snapshot prevents concurrent modification crash during file watcher events.
+- **STYLE**: ruff format applied to server.py.
+
+### v4.1.1 (2026-06-17)
+
+- **FIX**: All `_indexed_docs` iterations now use `list()` snapshot, preventing `dictionary changed size during iteration` crash when FileWatcher modifies the index concurrently with MCP tool calls (affects `search_knowledge`, `search_similar`, `update_document`, `remove_document`, `evaluate_retrieval`, `list_categories`, `list_documents`)
+
+### v4.1.0 (2026-06-17)
+
+- **Added:** `query_expansion_groups` config for symmetric synonym expansion (#92)
+- **Improved:** `expand_query()` now returns deterministic expansion order (set → ordered list with dedup)
+
+### v4.0.1 (2026-06-16)
+
+- **FIX**: Orphan cleanup now runs before indexing loop, preventing chunk loss when files are moved (#90).
+- **FIX**: Chunk deduplication is now per-document instead of global, preventing cross-document chunk deletion (#91).
+- **FIX**: Added `on_moved` handler to `DocumentWatcher` for proper file move detection.
+- **FIX**: Startup preflight probes ChromaDB in a child process and moves crashing persistent indexes to `data/backups/auto-repair-*` before MCP initialization.
+- **FIX**: Reranker load failures now fall back to RRF ordering instead of failing `search_knowledge` on offline machines.
+- **FIX**: Virtualenv project-root detection now handles Python symlinks that resolve to the system interpreter.
+- **NEW**: `knowledge-rag-guarded` console script kept as an explicit guarded startup alias.
+
 ### v4.0.0 (2026-06-09) — Enterprise Concurrent Access
 
 - **NEW**: SSE and streamable-http transport modes — 1 server serves N clients (`server.transport: "sse"` in config.yaml or `--transport sse` CLI).
@@ -1371,7 +1464,7 @@ Common issues:
 - **NEW**: ChromaDB WAL mode enabled automatically in SSE/HTTP mode for concurrent read performance.
 - **NEW**: Optional rate limiting — sliding-window counter, configurable RPM and burst, disabled by default.
 - **NEW**: Optional Prometheus metrics endpoint — tool call counts, latency histograms, separate port, disabled by default.
-- **NEW**: All 12 MCP tools instrumented with `@rate_limited` and `@instrument` decorators (zero-cost when disabled).
+- **NEW**: All 13 MCP tools instrumented with `@rate_limited` and `@instrument` decorators (zero-cost when disabled).
 - **NEW**: `--transport` CLI override for Docker/systemd deployments.
 - **NEW**: `pip install knowledge-rag[server]` optional dependency for SSE/HTTP (uvicorn).
 - **CHANGED**: SSE/HTTP mode auto-enables single-instance lock (port collision prevention).
@@ -1399,7 +1492,7 @@ Common issues:
 - **NEW** Property-based fuzzing of all parsers via Hypothesis (`tests/test_ingestion_property.py`) — 200 random examples per CI run.
 - **NEW** Memory baseline regression tests (`tests/test_memory_baseline.py`, cross-platform via psutil) — RSS bounded under 1000 queries; nightly soak amplifies to 50K iterations.
 - **NEW** Property/locale/format/preset matrices (`tests/test_presets.py`, `tests/test_locale.py`, `tests/test_format_smoke.py`).
-- **NEW** Backwards-compatibility regression tests (`tests/test_backwards_compat.py`) — legacy YAML configs from v3.6.0 / v3.7.0 still parse; all 12 MCP tool parameter names frozen.
+- **NEW** Backwards-compatibility regression tests (`tests/test_backwards_compat.py`) — legacy YAML configs from v3.6.0 / v3.7.0 still parse; all 13 MCP tool parameter names frozen.
 - **NEW** AST-based public API surface diff (`scripts/check_api_surface.py`) — any breaking change blocks merge, baseline at `.github/api-surface-baseline.json`.
 - **NEW** CHANGELOG enforcement (`scripts/check_changelog.py`) — user-facing PRs must add a bullet under `## Unreleased`; bypass via `skip-changelog` label.
 - **NEW** Test count anti-regression (`scripts/check_test_count.py`) — guards against silent test deletion.
@@ -1429,45 +1522,6 @@ Common issues:
 - **CHORE**: pytest `tmp_path_retention_count=1` to avoid Windows atexit cleanup race in CI.
 - **ROADMAP**: Tracked v4.0 shared-service architecture (one daemon, many thin MCP clients) as the long-term fix for multi-process resource duplication. (#34)
 
-### Unreleased
-
-### v4.2.0 (2026-06-17) — Search Performance & Output Quality
-
-- **PERF**: Custom inverted-index BM25 replaces `rank-bm25` full-corpus scan — 128× faster keyword search on 50K+ chunk corpora. Only documents containing query terms are scored via posting lists.
-- **PERF**: `numpy.argpartition` for O(n) top-k selection instead of O(n log n) sort.
-- **PERF**: Batched adjacent chunk fetch — single ChromaDB `collection.get()` call replaces N round-trips per result.
-- **PERF**: O(1) reverse lookup via `_source_to_docid` dict eliminates linear scans of `_indexed_docs` in `search_similar`, `update_document`, `remove_document`, and `_expand_with_adjacent_chunks`.
-- **NEW**: `snippet_mode` parameter on `search_knowledge` (default: `true`) — truncates content to ~500 chars at natural break points with `content_length` field. Reduces token consumption by ~72%.
-- **NEW**: `min_score` parameter on `search_knowledge` (default: `0.0`) — filters results below a normalized relevance threshold. Response includes `filtered_by_score` count.
-- **NEW**: `filtered_by_score` field in search response JSON for transparency.
-- **DEPS**: `numpy` added as direct dependency (was transitive via fastembed); `rank-bm25` import removed from server.py.
-- **TEST**: 6 new tests for `min_score` filtering and `snippet_mode` truncation.
-- **TEST**: Updated backwards-compat baseline to include new `search_knowledge` parameters.
-
-### v4.1.2 (2026-06-17)
-
-- **FIX**: `_save_metadata` dict snapshot prevents concurrent modification crash during file watcher events.
-- **STYLE**: ruff format applied to server.py.
-
-### v4.1.1 (2026-06-17)
-
-- **FIX**: All `_indexed_docs` iterations now use `list()` snapshot, preventing `dictionary changed size during iteration` crash when FileWatcher modifies the index concurrently with MCP tool calls (affects `search_knowledge`, `search_similar`, `update_document`, `remove_document`, `evaluate_retrieval`, `list_categories`, `list_documents`)
-
-### v4.1.0 (2026-06-17)
-
-- **Added:** `query_expansion_groups` config for symmetric synonym expansion (#92)
-- **Improved:** `expand_query()` now returns deterministic expansion order (set → ordered list with dedup)
-
-### v4.0.1 (2026-06-16)
-
-- **FIX**: Orphan cleanup now runs before indexing loop, preventing chunk loss when files are moved (#90).
-- **FIX**: Chunk deduplication is now per-document instead of global, preventing cross-document chunk deletion (#91).
-- **FIX**: Added `on_moved` handler to `DocumentWatcher` for proper file move detection.
-- **FIX**: Startup preflight probes ChromaDB in a child process and moves crashing persistent indexes to `data/backups/auto-repair-*` before MCP initialization.
-- **FIX**: Reranker load failures now fall back to RRF ordering instead of failing `search_knowledge` on offline machines.
-- **FIX**: Virtualenv project-root detection now handles Python symlinks that resolve to the system interpreter.
-- **NEW**: `knowledge-rag-guarded` console script kept as an explicit guarded startup alias.
-
 ### v3.6.2 (2026-04-23)
 
 - **INFRA**: NPM provenance attestation (SLSA supply chain security), full README on npm page

{knowledge_rag-4.2.0 → knowledge_rag-4.3.1}/mcp_server/__init__.py

@@ -8,7 +8,7 @@ import sys  # noqa: I001
 _original_stdout = sys.stdout
 sys.stdout = sys.stderr
 
-__version__ = "4.2.0"
+__version__ = "4.3.1"
 __author__ = "Ailton Rocha (Lyon.)"
 
 from .config import Config  # noqa: E402

{knowledge_rag-4.2.0 → knowledge_rag-4.3.1}/mcp_server/server.py

@@ -986,6 +986,9 @@ class KnowledgeOrchestrator:
         # Migration: deferred — checked in main() after full init
         self._needs_rebuild = False
 
+        # Background reindex progress (polled via get_index_stats)
+        self._reindex_progress: Dict[str, Any] = {"active": False}
+
     def _safe_get_collection(self):
         """
         Get or create ChromaDB collection with auto-recovery.
@@ -1134,6 +1137,7 @@ class KnowledgeOrchestrator:
 
         documents = self.parser.parse_directory()
         stats["total_files"] = len(documents)
+        self._reindex_progress["total_files"] = stats["total_files"]
         if stats["total_files"] > 100:
             print(f"[INDEX] Scanning {stats['total_files']} documents...")
 
@@ -1226,6 +1230,15 @@ class KnowledgeOrchestrator:
                 stats["errors"] += 1
                 print(f"[ERROR] Failed to index {doc.source}: {e}")
 
+            self._reindex_progress.update(
+                {
+                    "processed": idx + 1,
+                    "indexed": stats["indexed"],
+                    "skipped": stats["skipped"],
+                    "errors": stats["errors"],
+                }
+            )
+
             if stats["total_files"] > 100 and (idx + 1) % _progress_interval == 0:
                 pct = int((idx + 1) / stats["total_files"] * 100)
                 print(
@@ -1308,6 +1321,43 @@ class KnowledgeOrchestrator:
 
         return 0
 
+    def start_reindex_background(self, mode: str) -> Dict[str, Any]:
+        """Start reindex in a background thread. Returns immediately."""
+        if self._reindex_progress.get("active"):
+            return {"status": "already_running", "progress": dict(self._reindex_progress)}
+
+        self._reindex_progress = {
+            "active": True,
+            "operation": mode,
+            "total_files": 0,
+            "processed": 0,
+            "indexed": 0,
+            "skipped": 0,
+            "errors": 0,
+            "started_at": datetime.now().isoformat(),
+        }
+
+        target = {
+            "incremental": lambda: self.index_all(force=False),
+            "smart_reindex": self.reindex_all,
+            "nuclear_rebuild": self.nuclear_rebuild,
+        }[mode]
+
+        thread = threading.Thread(target=self._run_reindex, args=(target,), daemon=True)
+        thread.start()
+        return {"status": "started", "operation": mode}
+
+    def _run_reindex(self, target: Any) -> None:
+        """Background thread runner for reindex operations."""
+        try:
+            result = target()
+            self._reindex_progress["result"] = result
+        except Exception as e:
+            self._reindex_progress["error"] = str(e)
+            print(f"[ERROR] Background reindex failed: {e}")
+        finally:
+            self._reindex_progress["active"] = False
+
     def reindex_all(self) -> Dict[str, Any]:
         """Smart reindex: incremental detection + BM25 rebuild + orphan cleanup."""
         import shutil
@@ -1493,9 +1543,16 @@ class KnowledgeOrchestrator:
             else:
                 try:
                     fetched = self.collection.get(ids=[chunk_id], include=["documents", "metadatas"])
+                    if (
+                        not fetched["documents"]
+                        or not fetched["metadatas"]
+                        or not fetched["documents"][0]
+                        or not fetched["metadatas"][0]
+                    ):
+                        continue
                     data = {
-                        "document": fetched["documents"][0] if fetched["documents"] else "",
-                        "metadata": fetched["metadatas"][0] if fetched["metadatas"] else {},
+                        "document": fetched["documents"][0],
+                        "metadata": fetched["metadatas"][0],
                         "distance": 0,
                     }
                 except Exception:
@@ -2043,8 +2100,8 @@ class KnowledgeOrchestrator:
         return docs
 
     def get_stats(self) -> Dict[str, Any]:
-        """Get index statistics"""
-        return {
+        """Get index statistics including background reindex progress."""
+        stats = {
             "total_documents": len(self._indexed_docs),
             "total_chunks": self.collection.count(),
             "categories": self.list_categories(),
@@ -2057,6 +2114,48 @@ class KnowledgeOrchestrator:
             "query_cache": self.query_cache.stats(),
         }
 
+        progress = self._reindex_progress
+        if progress.get("active"):
+            total = max(1, progress.get("total_files", 1))
+            processed = progress.get("processed", 0)
+            stats["reindex"] = {
+                "active": True,
+                "operation": progress.get("operation"),
+                "progress": f"{processed}/{progress.get('total_files', 0)}",
+                "percent": round(processed / total * 100),
+                "indexed": progress.get("indexed", 0),
+                "errors": progress.get("errors", 0),
+                "started_at": progress.get("started_at"),
+            }
+        else:
+            stats["reindex"] = {"active": False}
+
+        return stats
+
+    def get_reindex_status(self) -> Dict[str, Any]:
+        """Get background reindex progress without computing full index stats."""
+        progress = self._reindex_progress
+        if progress.get("active"):
+            total = max(1, progress.get("total_files", 1))
+            processed = progress.get("processed", 0)
+            return {
+                "active": True,
+                "operation": progress.get("operation"),
+                "progress": f"{processed}/{progress.get('total_files', 0)}",
+                "percent": round(processed / total * 100),
+                "indexed": progress.get("indexed", 0),
+                "skipped": progress.get("skipped", 0),
+                "errors": progress.get("errors", 0),
+                "started_at": progress.get("started_at"),
+            }
+
+        result: Dict[str, Any] = {"active": False}
+        if "result" in progress:
+            result["last_result"] = progress["result"]
+        if "error" in progress:
+            result["last_error"] = progress["error"]
+        return result
+
     def _load_metadata(self) -> Dict[str, Dict]:
         """Load index metadata from disk"""
         if self._metadata_file.exists():
@@ -2181,7 +2280,7 @@ def search_knowledge(
     hybrid_alpha = max(0.0, min(hybrid_alpha if hybrid_alpha is not None else 0.3, 1.0))
     min_score = max(0.0, min(min_score if min_score is not None else 0.0, 1.0))
 
-    valid_categories = list(config.keyword_routes.keys()) + list(set(config.category_mappings.values()))
+    valid_categories = list(config.keyword_routes.keys()) + list(set(config.category_mappings.values())) + ["general"]
     if category and category not in valid_categories:
         return json.dumps(
             {"status": "error", "message": f"Invalid category '{category}'. Valid: {', '.join(valid_categories)}"}
@@ -2258,16 +2357,17 @@ def reindex_documents(force: bool = False, full_rebuild: bool = False) -> str:
     """
     Index or reindex all documents in the knowledge base.
 
-    Mutating — modifies the vector index. CPU/IO intensive for full_rebuild (~6 min for 200 docs).
+    Runs in background — returns immediately. Use get_reindex_status() to monitor progress.
 
     Args:
-        force: If True, smart reindex (detects changed files + rebuilds BM25 index). Fast (~5s
-            for 200 docs). Use after manually editing files on disk outside of add_document().
+        force: If True, smart reindex (detects changed files + rebuilds BM25 index).
+            Use after manually editing files on disk outside of add_document().
         full_rebuild: If True, nuclear rebuild — deletes all vectors and re-embeds everything
             from scratch. Use only if the embedding model changed or the index is corrupted.
 
     Returns:
-        JSON string with indexing statistics (docs processed, added, skipped, errors).
+        JSON string with operation status. Poll get_reindex_status() for reindex.active,
+        reindex.progress, and reindex.percent until reindex.active becomes false.
 
     Usage: Normal workflow does not require this — add_document(), update_document(), and
     add_from_url() all auto-index on call. Use force=True only after direct filesystem edits.
@@ -2277,16 +2377,59 @@ def reindex_documents(force: bool = False, full_rebuild: bool = False) -> str:
     orchestrator = get_orchestrator()
 
     if full_rebuild:
-        stats = orchestrator.nuclear_rebuild()
-        operation = "nuclear_rebuild"
+        mode = "nuclear_rebuild"
     elif force:
-        stats = orchestrator.reindex_all()
-        operation = "smart_reindex"
+        mode = "smart_reindex"
     else:
-        stats = orchestrator.index_all()
-        operation = "incremental_index"
+        mode = "incremental"
+
+    result = orchestrator.start_reindex_background(mode)
+
+    if result["status"] == "already_running":
+        progress = result["progress"]
+        return json.dumps(
+            {
+                "status": "already_running",
+                "progress": f"{progress.get('processed', 0)}/{progress.get('total_files', 0)}",
+                "operation": progress.get("operation"),
+                "hint": "Use get_reindex_status() to check progress",
+            },
+            indent=2,
+            ensure_ascii=False,
+        )
+
+    return json.dumps(
+        {
+            "status": "started",
+            "operation": mode,
+            "message": "Reindex running in background. Use get_reindex_status() to monitor progress.",
+        },
+        indent=2,
+        ensure_ascii=False,
+    )
+
 
-    return json.dumps({"status": "success", "operation": operation, "stats": stats}, indent=2, ensure_ascii=False)
+@mcp.tool()
+@rate_limited
+@instrument("get_reindex_status")
+def get_reindex_status() -> str:
+    """
+    Get the current status of a background reindex operation.
+
+    Lightweight — does not compute full index statistics. Use this to poll progress
+    after calling reindex_documents().
+
+    Returns:
+        JSON string with reindex status. When active: operation name, progress (processed/total),
+        percent complete, indexed/skipped/errors counts, and start time. When inactive: active=false,
+        plus last_result or last_error from the most recent completed reindex.
+
+    Usage: Call repeatedly after reindex_documents() to monitor progress. When reindex.active
+    becomes false, the operation is complete. Use get_index_stats() for full index health metrics.
+    """
+    orchestrator = get_orchestrator()
+    status = orchestrator.get_reindex_status()
+    return json.dumps({"status": "success", "reindex": status}, indent=2)
 
 
 @mcp.tool()

{knowledge_rag-4.2.0 → knowledge_rag-4.3.1}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
 
 [project]
 name = "knowledge-rag"
-version = "4.2.0"
-description = "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 12 MCP Tools + 20 Format Parsers. Zero external servers."
+version = "4.3.1"
+description = "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers."
 readme = "README.md"
 license = {text = "MIT"}
 requires-python = ">=3.11"
@@ -44,7 +44,17 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-gpu = ["onnxruntime-gpu>=1.14.0"]
+gpu = [
+    "onnxruntime-gpu>=1.14.0",
+    "nvidia-cublas-cu12",
+    "nvidia-cudnn-cu12",
+    "nvidia-cuda-runtime-cu12",
+    "nvidia-cufft-cu12",
+    "nvidia-cusparse-cu12",
+    "nvidia-cusolver-cu12",
+    "nvidia-curand-cu12",
+    "nvidia-nvjitlink-cu12",
+]
 server = ["uvicorn>=0.20.0"]
 
 [project.urls]
@@ -131,7 +141,11 @@ fail_under = 35
 # we incrementally annotate the legacy modules. The CI job runs strict on the
 # allowlist below; new modules are added as they earn full annotations.
 [tool.mypy]
-python_version = "3.11"
+# Pinned to 3.12 to match the CI runtime (Python 3.12) and to accept PEP 695
+# ``type`` statements that recent third-party stubs (e.g. numpy/__init__.pyi)
+# emit. The package itself still supports Python 3.11+ at runtime — this
+# setting only governs the static-analysis target.
+python_version = "3.12"
 strict = true
 show_error_codes = true
 warn_unused_configs = true