knowledge-rag 4.8.3__tar.gz → 4.8.4__tar.gz

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: knowledge-rag
-Version: 4.8.3
+Version: 4.8.4
 Summary: Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers.
 Project-URL: Homepage, https://github.com/lyonzin/knowledge-rag
 Project-URL: Repository, https://github.com/lyonzin/knowledge-rag
@@ -278,19 +278,32 @@ See [Changelog](#changelog) for full history.
 flowchart TB
     subgraph MCP["MCP SERVER (FastMCP)"]
         direction TB
-        TOOLS["13 MCP Tools<br/>search | get | add | update | remove<br/>reindex | reindex_status | list | stats | url | similar | evaluate"]
+        TOOLS["13 MCP Tools (frozen, LEI 1)<br/>search_knowledge | get_document | search_similar<br/>add_document | add_from_url | update_document | remove_document<br/>reindex_documents | get_reindex_status<br/>list_categories | list_documents | get_index_stats | evaluate_retrieval"]
     end
 
-    subgraph SEARCH["HYBRID SEARCH ENGINE"]
+    subgraph SEARCH["SEARCH DISPATCH (v4.8.2+)"]
+        direction TB
+        SM["search_method dispatch<br/>auto | hybrid | fts5"]
+        QROUTER["QueryRouter (regex)<br/>lexical vs semantic<br/>(ADR-002)"]
+        FTS5PATH["FTS5 Fast-Path<br/>(exact identifiers <10ms)"]
+        HYBRIDPATH["Hybrid Pipeline<br/>(conceptual queries)"]
+
+        SM --> QROUTER
+        QROUTER -->|lexical| FTS5PATH
+        QROUTER -->|semantic| HYBRIDPATH
+        FTS5PATH -.->|not_ready / low_hits<br/>fallback| HYBRIDPATH
+    end
+
+    subgraph HYBRID["HYBRID PIPELINE"]
         direction LR
-        ROUTER["Keyword Router<br/>(word boundaries)"]
+        KWROUTER["Keyword Router<br/>(word boundaries → category filter)"]
         SEMANTIC["Semantic Search<br/>(ChromaDB)"]
         BM25["BM25 Keyword<br/>(inverted-index + expansion)"]
         RRF["Reciprocal Rank<br/>Fusion (RRF)"]
         RERANK["Cross-Encoder<br/>Reranker"]
 
-        ROUTER --> SEMANTIC
-        ROUTER --> BM25
+        KWROUTER --> SEMANTIC
+        KWROUTER --> BM25
         SEMANTIC --> RRF
         BM25 --> RRF
         RRF --> RERANK
@@ -298,9 +311,11 @@ flowchart TB
 
     subgraph STORAGE["STORAGE LAYER"]
         direction LR
-        CHROMA[("ChromaDB<br/>Vector Database")]
-        COLLECTIONS["Collections<br/>security | ctf<br/>logscale | development"]
+        CHROMA[("ChromaDB<br/>data/chroma_db/<br/>(vector store)")]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(lexical index, ADR-001)")]
+        COLLECTIONS["8 Categories<br/>redteam | blueteam | ctf<br/>security | logscale<br/>development | aar | general"]
         CHROMA --- COLLECTIONS
+        FTS5DB --- COLLECTIONS
     end
 
     subgraph EMBED["EMBEDDINGS (In-Process)"]
@@ -312,36 +327,55 @@ flowchart TB
     subgraph INGEST["DOCUMENT INGESTION"]
         PARSERS["20 Parsers<br/>MD | PDF | TXT | PY | C | H | CPP | JS | JSX | TS | TSX | JSON | XML | CSV<br/>DOCX | XLSX | PPTX | IPYNB | MQH | MQ4"]
         CHUNKER["Chunking<br/>MD: section-aware<br/>Other: 1000 chars + 200 overlap"]
-        PARSERS --> CHUNKER
+        FTS5SYNC["FTS5 CRUD sync<br/>(v4.8.2 task 05)"]
+        PARSERS --> CHUNKER --> FTS5SYNC
     end
 
     CLAUDE["Claude Code"] --> MCP
     MCP --> SEARCH
-    SEARCH --> STORAGE
+    HYBRIDPATH --> HYBRID
+    FTS5PATH --> FTS5DB
+    HYBRID --> STORAGE
     STORAGE --> EMBED
     INGEST --> EMBED
     EMBED --> STORAGE
+    FTS5SYNC --> FTS5DB
 ```
 
 ### Query Processing Flow
 
 ```mermaid
 flowchart TB
-    QUERY["User Query<br/>'mimikatz credential dump'"] --> EXPAND
+    QUERY["User Query<br/>'mimikatz credential dump' | 'CVE-2021-4034'"] --> METHOD
+
+    subgraph DISPATCH["Dispatch (v4.8.2+, ADR-006)"]
+        METHOD{"search_method<br/>auto | hybrid | fts5"}
+        AUTOROUTE["QueryRouter.classify()<br/>regex-based lexical detection"]
+        FASTPATH["FTS5 Fast-Path<br/>SQLite MATCH<br/>(&lt;10ms cold, &lt;2ms hot)"]
+        NOTREADY{"FTS5 ready<br/>AND hits >= min_hits?"}
+
+        METHOD -->|auto| AUTOROUTE
+        METHOD -->|fts5| FASTPATH
+        METHOD -->|hybrid| EXPAND
+        AUTOROUTE -->|lexical| FASTPATH
+        AUTOROUTE -->|semantic| EXPAND
+        FASTPATH --> NOTREADY
+        NOTREADY -->|no, fallback| EXPAND
+    end
 
-    subgraph EXPANSION["Query Expansion"]
+    subgraph EXPANSION["Query Expansion (hybrid path)"]
         EXPAND["Synonym Expansion<br/>mimikatz -> mimikatz, sekurlsa, logonpasswords"]
     end
 
-    EXPAND --> ROUTER
+    EXPAND --> KWROUTER
 
-    subgraph ROUTING["Keyword Routing"]
-        ROUTER["Keyword Router"]
+    subgraph ROUTING["Keyword Routing (category filter)"]
+        KWROUTER["Keyword Router<br/>(word boundaries)"]
         MATCH{"Word Boundary<br/>Match?"}
         CATEGORY["Filter: redteam"]
         NOFILTER["No Filter"]
 
-        ROUTER --> MATCH
+        KWROUTER --> MATCH
         MATCH -->|Yes| CATEGORY
         MATCH -->|No| NOFILTER
     end
@@ -374,7 +408,8 @@ flowchart TB
     BM25 --> RRF
 
     ADJ --> MINSCORE
-    SNIPPET --> RESULTS["Results<br/>search_method: hybrid|semantic|keyword<br/>score + filtered_by_score + content_length"]
+    NOTREADY -->|yes, hit| MINSCORE
+    SNIPPET --> RESULTS["Results<br/>search_method: fts5 | hybrid | semantic | keyword<br/>routed_by: fts5_router | none<br/>score + filtered_by_score + content_length"]
 ```
 
 ### Document Ingestion Flow
@@ -382,7 +417,7 @@ flowchart TB
 ```mermaid
 flowchart LR
     subgraph INPUT["Input"]
-        FILES["documents/<br/>├── security/<br/>├── development/<br/>├── ctf/<br/>└── general/"]
+        FILES["documents/<br/>├── security/<br/>│   ├── redteam/<br/>│   ├── blueteam/<br/>│   └── ctf/<br/>├── aar/<br/>├── logscale/<br/>├── development/<br/>└── general/"]
     end
 
     subgraph PARSE["Parse (20 formats)"]
@@ -402,9 +437,18 @@ flowchart LR
         FASTEMBED["FastEmbed ONNX<br/>bge-small-en-v1.5<br/>(384D, CPU or GPU)"]
     end
 
+    subgraph DISPATCH["Write Dispatch (v4.8.3, #161)"]
+        WCOL{"_write_collection<br/>routes writes"}
+        STAGING["Staging Collection<br/>(nuclear_rebuild only,<br/>keeps prod serving queries)"]
+        PROD["Production Collection<br/>(default)"]
+        WCOL -->|_staging_target set| STAGING
+        WCOL -->|default| PROD
+    end
+
     subgraph STORE["Store"]
-        CHROMADB[("ChromaDB")]
-        BM25IDX["BM25 Index"]
+        CHROMADB[("ChromaDB<br/>data/chroma_db/")]
+        BM25IDX["BM25 Index<br/>(in-memory,<br/>rebuilt on write)"]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(CRUD-synced, ADR-008)")]
     end
 
     FILES --> MD & PDF & OFFICE & CODE
@@ -413,14 +457,20 @@ flowchart LR
     MDSPLIT --> DEDUP
     TXTSPLIT --> DEDUP
     DEDUP --> EMBED
-    EMBED --> STORE
+    EMBED --> DISPATCH
+    STAGING --> CHROMADB
+    PROD --> CHROMADB
+    PROD --> BM25IDX
+    PROD --> FTS5DB
 ```
 
 ### hybrid_alpha Parameter Effect
 
+`hybrid_alpha` weights RRF fusion between semantic and BM25 rankings on the **hybrid pipeline only**. When `search_method="auto"` and the QueryRouter classifies the query as lexical (e.g. `CVE-2021-4034`, `MDR-AD002`, `T1078.001`, file hashes), the FTS5 fast-path fires and `hybrid_alpha` is not consulted — FTS5 uses SQLite's native `bm25()` scoring exclusively. Pass `search_method="hybrid"` to force RRF fusion + rerank on every query if you want deterministic hybrid semantics regardless of the query shape.
+
 ```mermaid
 flowchart LR
-    subgraph ALPHA["hybrid_alpha values"]
+    subgraph ALPHA["hybrid_alpha values (hybrid path only)"]
         A0["0.0<br/>Pure BM25<br/>Instant"]
         A3["0.3 (default)<br/>Keyword-heavy<br/>Fast"]
         A5["0.5<br/>Balanced"]
@@ -429,7 +479,7 @@ flowchart LR
     end
 
     subgraph USE["Best For"]
-        U0["CVEs, tool names<br/>exact matches"]
+        U0["CVEs, tool names<br/>exact matches<br/>(consider FTS5 fast-path)"]
         U3["Technical queries<br/>specific terms"]
         U5["General queries"]
         U7["Conceptual queries<br/>related topics"]
@@ -1483,6 +1533,23 @@ Common issues:
 
 ### Unreleased
 
+### v4.8.4 (2026-08-13) — Patch: security + durability + defensive fixes
+
+**Recommended upgrade for anyone running HTTP/SSE transport with bearer auth on a non-loopback host, or relying on `reindex_documents(force=True)`.** Four independent fixes accumulated since v4.8.3: one previously blocked remote-with-auth deployments (#174), one silently corrupted durable index metadata after a failed rebuild (#173), and two defensive fixes hardened `.ipynb` ingestion (#175) and the nightly chaos suite (#164).
+
+**Fixed:**
+
+- **fix(security)** — create the HTTP MCP app with the configured `server_host`. On the bearer-auth path the app factory was called without a host, so it defaulted to `127.0.0.1` and MCP 2.0.0's localhost-only DNS-rebinding allowlist rejected a configured non-loopback host with `421 Invalid Host header` before auth ran — even though uvicorn was bound to that host. Remote/LAN deployments with auth are now reachable (#174).
+- **fix(reindex)** — restore durable metadata on a failed zero-downtime rebuild. The staging populate path persisted the staging map to `index_metadata.json` before validation ran; when validation failed, rollback restored only the in-memory `_indexed_docs`, leaving the file describing staging while memory held production. After a restart `_load_metadata` trusted the stale file, corrupting skip/update and embedding-dimension decisions. Rollback now re-persists the restored production metadata (#173).
+- **fix(ingestion)** — `_parse_ipynb` handled invalid JSON gracefully but then assumed the parsed notebook was a well-formed dict. A valid-JSON notebook with a null `metadata` or `cells` field, a non-dict cell, a bare list, or a dict cell whose `source` is not `str | list[str]` (a number, a dict, or a list with non-string items) crashed with `AttributeError`/`TypeError`. Guarded the notebook, `metadata`, `cells`, cell, and cell `source` lookups so a parseable-but-malformed notebook is handled the same as any other odd input (#175, thanks @eeshsaxena).
+- **fix(ci)** — the nightly chaos job (`pytest -m chaos`) aborted collection with `ModuleNotFoundError: No module named 'hypothesis'` because `tests/test_query_router.py` imported hypothesis at module top-level while the chaos job installs only `pytest psutil numpy`. Guarded the import with `pytest.importorskip("hypothesis")` so the module is skipped cleanly instead of failing the whole nightly run (#164).
+
+**Docs:**
+
+- **docs(architecture)** — sync the 4 Mermaid diagrams in `## Architecture` (System Overview, Query Processing Flow, Document Ingestion Flow, hybrid_alpha Parameter Effect) with the v4.8.2 FTS5 fast-path (ADR-002/003/006), the v4.8.3 `_write_collection` staging dispatcher (#161), and the current 8-category layout. The diagrams previously described only the pre-v4.8 hybrid pipeline and 4-category storage; they now also show FTS5 dispatch, fallback semantics, FTS5 CRUD sync (ADR-008), and the zero-downtime staging write path. `hybrid_alpha` now clarifies that the fast-path bypasses RRF entirely so the weighting only matters on the hybrid path.
+
+**No breaking changes.** Backwards compat verified by `check_api_surface.py` (zero diff vs baseline). All 13 MCP tool parameter names frozen (verified by `tests/test_backwards_compat.py`).
+
 ### v4.8.3 (2026-08-10) — Critical hotfix: nuclear-rebuild + smart-reindex hardening
 
 **Recommended upgrade for any corpus with >1000 chunks or any user relying on `reindex_documents(force=True)` after a config change.** The v4.8.0 nuclear rebuild + v4.8.2 FTS5 fast-path have several latent state-management bugs that only trigger under a real-world combination (large corpus + rebuild + concurrent queries + FTS5 opt-in). Enterprise users with hacktricks-scale corpora (~50k+ chunks) are affected.

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/README.md

@@ -230,19 +230,32 @@ See [Changelog](#changelog) for full history.
 flowchart TB
     subgraph MCP["MCP SERVER (FastMCP)"]
         direction TB
-        TOOLS["13 MCP Tools<br/>search | get | add | update | remove<br/>reindex | reindex_status | list | stats | url | similar | evaluate"]
+        TOOLS["13 MCP Tools (frozen, LEI 1)<br/>search_knowledge | get_document | search_similar<br/>add_document | add_from_url | update_document | remove_document<br/>reindex_documents | get_reindex_status<br/>list_categories | list_documents | get_index_stats | evaluate_retrieval"]
     end
 
-    subgraph SEARCH["HYBRID SEARCH ENGINE"]
+    subgraph SEARCH["SEARCH DISPATCH (v4.8.2+)"]
+        direction TB
+        SM["search_method dispatch<br/>auto | hybrid | fts5"]
+        QROUTER["QueryRouter (regex)<br/>lexical vs semantic<br/>(ADR-002)"]
+        FTS5PATH["FTS5 Fast-Path<br/>(exact identifiers <10ms)"]
+        HYBRIDPATH["Hybrid Pipeline<br/>(conceptual queries)"]
+
+        SM --> QROUTER
+        QROUTER -->|lexical| FTS5PATH
+        QROUTER -->|semantic| HYBRIDPATH
+        FTS5PATH -.->|not_ready / low_hits<br/>fallback| HYBRIDPATH
+    end
+
+    subgraph HYBRID["HYBRID PIPELINE"]
         direction LR
-        ROUTER["Keyword Router<br/>(word boundaries)"]
+        KWROUTER["Keyword Router<br/>(word boundaries → category filter)"]
         SEMANTIC["Semantic Search<br/>(ChromaDB)"]
         BM25["BM25 Keyword<br/>(inverted-index + expansion)"]
         RRF["Reciprocal Rank<br/>Fusion (RRF)"]
         RERANK["Cross-Encoder<br/>Reranker"]
 
-        ROUTER --> SEMANTIC
-        ROUTER --> BM25
+        KWROUTER --> SEMANTIC
+        KWROUTER --> BM25
         SEMANTIC --> RRF
         BM25 --> RRF
         RRF --> RERANK
@@ -250,9 +263,11 @@ flowchart TB
 
     subgraph STORAGE["STORAGE LAYER"]
         direction LR
-        CHROMA[("ChromaDB<br/>Vector Database")]
-        COLLECTIONS["Collections<br/>security | ctf<br/>logscale | development"]
+        CHROMA[("ChromaDB<br/>data/chroma_db/<br/>(vector store)")]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(lexical index, ADR-001)")]
+        COLLECTIONS["8 Categories<br/>redteam | blueteam | ctf<br/>security | logscale<br/>development | aar | general"]
         CHROMA --- COLLECTIONS
+        FTS5DB --- COLLECTIONS
     end
 
     subgraph EMBED["EMBEDDINGS (In-Process)"]
@@ -264,36 +279,55 @@ flowchart TB
     subgraph INGEST["DOCUMENT INGESTION"]
         PARSERS["20 Parsers<br/>MD | PDF | TXT | PY | C | H | CPP | JS | JSX | TS | TSX | JSON | XML | CSV<br/>DOCX | XLSX | PPTX | IPYNB | MQH | MQ4"]
         CHUNKER["Chunking<br/>MD: section-aware<br/>Other: 1000 chars + 200 overlap"]
-        PARSERS --> CHUNKER
+        FTS5SYNC["FTS5 CRUD sync<br/>(v4.8.2 task 05)"]
+        PARSERS --> CHUNKER --> FTS5SYNC
     end
 
     CLAUDE["Claude Code"] --> MCP
     MCP --> SEARCH
-    SEARCH --> STORAGE
+    HYBRIDPATH --> HYBRID
+    FTS5PATH --> FTS5DB
+    HYBRID --> STORAGE
     STORAGE --> EMBED
     INGEST --> EMBED
     EMBED --> STORAGE
+    FTS5SYNC --> FTS5DB
 ```
 
 ### Query Processing Flow
 
 ```mermaid
 flowchart TB
-    QUERY["User Query<br/>'mimikatz credential dump'"] --> EXPAND
+    QUERY["User Query<br/>'mimikatz credential dump' | 'CVE-2021-4034'"] --> METHOD
+
+    subgraph DISPATCH["Dispatch (v4.8.2+, ADR-006)"]
+        METHOD{"search_method<br/>auto | hybrid | fts5"}
+        AUTOROUTE["QueryRouter.classify()<br/>regex-based lexical detection"]
+        FASTPATH["FTS5 Fast-Path<br/>SQLite MATCH<br/>(&lt;10ms cold, &lt;2ms hot)"]
+        NOTREADY{"FTS5 ready<br/>AND hits >= min_hits?"}
+
+        METHOD -->|auto| AUTOROUTE
+        METHOD -->|fts5| FASTPATH
+        METHOD -->|hybrid| EXPAND
+        AUTOROUTE -->|lexical| FASTPATH
+        AUTOROUTE -->|semantic| EXPAND
+        FASTPATH --> NOTREADY
+        NOTREADY -->|no, fallback| EXPAND
+    end
 
-    subgraph EXPANSION["Query Expansion"]
+    subgraph EXPANSION["Query Expansion (hybrid path)"]
         EXPAND["Synonym Expansion<br/>mimikatz -> mimikatz, sekurlsa, logonpasswords"]
     end
 
-    EXPAND --> ROUTER
+    EXPAND --> KWROUTER
 
-    subgraph ROUTING["Keyword Routing"]
-        ROUTER["Keyword Router"]
+    subgraph ROUTING["Keyword Routing (category filter)"]
+        KWROUTER["Keyword Router<br/>(word boundaries)"]
         MATCH{"Word Boundary<br/>Match?"}
         CATEGORY["Filter: redteam"]
         NOFILTER["No Filter"]
 
-        ROUTER --> MATCH
+        KWROUTER --> MATCH
         MATCH -->|Yes| CATEGORY
         MATCH -->|No| NOFILTER
     end
@@ -326,7 +360,8 @@ flowchart TB
     BM25 --> RRF
 
     ADJ --> MINSCORE
-    SNIPPET --> RESULTS["Results<br/>search_method: hybrid|semantic|keyword<br/>score + filtered_by_score + content_length"]
+    NOTREADY -->|yes, hit| MINSCORE
+    SNIPPET --> RESULTS["Results<br/>search_method: fts5 | hybrid | semantic | keyword<br/>routed_by: fts5_router | none<br/>score + filtered_by_score + content_length"]
 ```
 
 ### Document Ingestion Flow
@@ -334,7 +369,7 @@ flowchart TB
 ```mermaid
 flowchart LR
     subgraph INPUT["Input"]
-        FILES["documents/<br/>├── security/<br/>├── development/<br/>├── ctf/<br/>└── general/"]
+        FILES["documents/<br/>├── security/<br/>│   ├── redteam/<br/>│   ├── blueteam/<br/>│   └── ctf/<br/>├── aar/<br/>├── logscale/<br/>├── development/<br/>└── general/"]
     end
 
     subgraph PARSE["Parse (20 formats)"]
@@ -354,9 +389,18 @@ flowchart LR
         FASTEMBED["FastEmbed ONNX<br/>bge-small-en-v1.5<br/>(384D, CPU or GPU)"]
     end
 
+    subgraph DISPATCH["Write Dispatch (v4.8.3, #161)"]
+        WCOL{"_write_collection<br/>routes writes"}
+        STAGING["Staging Collection<br/>(nuclear_rebuild only,<br/>keeps prod serving queries)"]
+        PROD["Production Collection<br/>(default)"]
+        WCOL -->|_staging_target set| STAGING
+        WCOL -->|default| PROD
+    end
+
     subgraph STORE["Store"]
-        CHROMADB[("ChromaDB")]
-        BM25IDX["BM25 Index"]
+        CHROMADB[("ChromaDB<br/>data/chroma_db/")]
+        BM25IDX["BM25 Index<br/>(in-memory,<br/>rebuilt on write)"]
+        FTS5DB[("SQLite FTS5<br/>data/fts5_index.db<br/>(CRUD-synced, ADR-008)")]
     end
 
     FILES --> MD & PDF & OFFICE & CODE
@@ -365,14 +409,20 @@ flowchart LR
     MDSPLIT --> DEDUP
     TXTSPLIT --> DEDUP
     DEDUP --> EMBED
-    EMBED --> STORE
+    EMBED --> DISPATCH
+    STAGING --> CHROMADB
+    PROD --> CHROMADB
+    PROD --> BM25IDX
+    PROD --> FTS5DB
 ```
 
 ### hybrid_alpha Parameter Effect
 
+`hybrid_alpha` weights RRF fusion between semantic and BM25 rankings on the **hybrid pipeline only**. When `search_method="auto"` and the QueryRouter classifies the query as lexical (e.g. `CVE-2021-4034`, `MDR-AD002`, `T1078.001`, file hashes), the FTS5 fast-path fires and `hybrid_alpha` is not consulted — FTS5 uses SQLite's native `bm25()` scoring exclusively. Pass `search_method="hybrid"` to force RRF fusion + rerank on every query if you want deterministic hybrid semantics regardless of the query shape.
+
 ```mermaid
 flowchart LR
-    subgraph ALPHA["hybrid_alpha values"]
+    subgraph ALPHA["hybrid_alpha values (hybrid path only)"]
         A0["0.0<br/>Pure BM25<br/>Instant"]
         A3["0.3 (default)<br/>Keyword-heavy<br/>Fast"]
         A5["0.5<br/>Balanced"]
@@ -381,7 +431,7 @@ flowchart LR
     end
 
     subgraph USE["Best For"]
-        U0["CVEs, tool names<br/>exact matches"]
+        U0["CVEs, tool names<br/>exact matches<br/>(consider FTS5 fast-path)"]
         U3["Technical queries<br/>specific terms"]
         U5["General queries"]
         U7["Conceptual queries<br/>related topics"]
@@ -1435,6 +1485,23 @@ Common issues:
 
 ### Unreleased
 
+### v4.8.4 (2026-08-13) — Patch: security + durability + defensive fixes
+
+**Recommended upgrade for anyone running HTTP/SSE transport with bearer auth on a non-loopback host, or relying on `reindex_documents(force=True)`.** Four independent fixes accumulated since v4.8.3: one previously blocked remote-with-auth deployments (#174), one silently corrupted durable index metadata after a failed rebuild (#173), and two defensive fixes hardened `.ipynb` ingestion (#175) and the nightly chaos suite (#164).
+
+**Fixed:**
+
+- **fix(security)** — create the HTTP MCP app with the configured `server_host`. On the bearer-auth path the app factory was called without a host, so it defaulted to `127.0.0.1` and MCP 2.0.0's localhost-only DNS-rebinding allowlist rejected a configured non-loopback host with `421 Invalid Host header` before auth ran — even though uvicorn was bound to that host. Remote/LAN deployments with auth are now reachable (#174).
+- **fix(reindex)** — restore durable metadata on a failed zero-downtime rebuild. The staging populate path persisted the staging map to `index_metadata.json` before validation ran; when validation failed, rollback restored only the in-memory `_indexed_docs`, leaving the file describing staging while memory held production. After a restart `_load_metadata` trusted the stale file, corrupting skip/update and embedding-dimension decisions. Rollback now re-persists the restored production metadata (#173).
+- **fix(ingestion)** — `_parse_ipynb` handled invalid JSON gracefully but then assumed the parsed notebook was a well-formed dict. A valid-JSON notebook with a null `metadata` or `cells` field, a non-dict cell, a bare list, or a dict cell whose `source` is not `str | list[str]` (a number, a dict, or a list with non-string items) crashed with `AttributeError`/`TypeError`. Guarded the notebook, `metadata`, `cells`, cell, and cell `source` lookups so a parseable-but-malformed notebook is handled the same as any other odd input (#175, thanks @eeshsaxena).
+- **fix(ci)** — the nightly chaos job (`pytest -m chaos`) aborted collection with `ModuleNotFoundError: No module named 'hypothesis'` because `tests/test_query_router.py` imported hypothesis at module top-level while the chaos job installs only `pytest psutil numpy`. Guarded the import with `pytest.importorskip("hypothesis")` so the module is skipped cleanly instead of failing the whole nightly run (#164).
+
+**Docs:**
+
+- **docs(architecture)** — sync the 4 Mermaid diagrams in `## Architecture` (System Overview, Query Processing Flow, Document Ingestion Flow, hybrid_alpha Parameter Effect) with the v4.8.2 FTS5 fast-path (ADR-002/003/006), the v4.8.3 `_write_collection` staging dispatcher (#161), and the current 8-category layout. The diagrams previously described only the pre-v4.8 hybrid pipeline and 4-category storage; they now also show FTS5 dispatch, fallback semantics, FTS5 CRUD sync (ADR-008), and the zero-downtime staging write path. `hybrid_alpha` now clarifies that the fast-path bypasses RRF entirely so the weighting only matters on the hybrid path.
+
+**No breaking changes.** Backwards compat verified by `check_api_surface.py` (zero diff vs baseline). All 13 MCP tool parameter names frozen (verified by `tests/test_backwards_compat.py`).
+
 ### v4.8.3 (2026-08-10) — Critical hotfix: nuclear-rebuild + smart-reindex hardening
 
 **Recommended upgrade for any corpus with >1000 chunks or any user relying on `reindex_documents(force=True)` after a config change.** The v4.8.0 nuclear rebuild + v4.8.2 FTS5 fast-path have several latent state-management bugs that only trigger under a real-world combination (large corpus + rebuild + concurrent queries + FTS5 opt-in). Enterprise users with hacktricks-scale corpora (~50k+ chunks) are affected.

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/mcp_server/__init__.py

@@ -8,7 +8,7 @@ import sys  # noqa: I001
 _original_stdout = sys.stdout
 sys.stdout = sys.stderr
 
-__version__ = "4.8.3"
+__version__ = "4.8.4"
 __author__ = "Ailton Rocha (Lyon.)"
 
 from .config import Config  # noqa: E402

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/mcp_server/ingestion.py

@@ -613,23 +613,44 @@ class DocumentParser:
             metadata["is_valid_json"] = False
             return raw, metadata
 
+        # Valid JSON, but not necessarily a well-formed notebook: it may be a
+        # bare list/scalar, or have null / wrong-typed "metadata"/"cells" fields.
+        # Guard the lookups so a malformed-but-parseable .ipynb is handled
+        # gracefully instead of crashing with AttributeError/TypeError.
+        if not isinstance(nb, dict):
+            metadata["is_valid_json"] = False
+            return raw, metadata
+
         metadata["is_valid_json"] = True
         metadata["nbformat"] = nb.get("nbformat", 0)
-        kernel = nb.get("metadata", {}).get("kernelspec", {})
+        nb_metadata = nb.get("metadata")
+        kernel = nb_metadata.get("kernelspec") if isinstance(nb_metadata, dict) else None
+        if not isinstance(kernel, dict):
+            kernel = {}
         metadata["kernel"] = kernel.get("display_name", kernel.get("name", "unknown"))
 
-        cells = nb.get("cells", [])
+        cells = nb.get("cells")
+        if not isinstance(cells, list):
+            cells = []
         metadata["cells"] = len(cells)
         code_cells = 0
         markdown_cells = 0
 
         parts = []
         for cell in cells:
+            if not isinstance(cell, dict):
+                continue
             cell_type = cell.get("cell_type", "")
             source = cell.get("source", "")
 
+            # Per the nbformat spec `source` is str | list[str]. Anything else
+            # (a number, a dict, a list with non-string items) is the same
+            # malformed-but-parseable case handled above, so tolerate it rather
+            # than letting it reach "".join()/.strip() and raise.
             if isinstance(source, list):
-                source = "".join(source)
+                source = "".join(s for s in source if isinstance(s, str))
+            elif not isinstance(source, str):
+                continue
 
             if not source or not source.strip():
                 continue

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/mcp_server/server.py

@@ -2420,6 +2420,12 @@ class KnowledgeOrchestrator:
         # collection we're about to delete.
         self._staging_target = None
         self._rollback_staging_state(saved)
+        # GH #173: populate persisted the staging map to index_metadata.json
+        # (index_all -> _finalize_reindex -> _save_metadata) before validation.
+        # Restoring only in-memory fields would leave the file describing
+        # staging while memory holds production; re-persist so durable metadata
+        # matches the restored state and a restart won't load stale staging.
+        self._save_metadata()
         try:
             self.chroma_client.delete_collection(f"{prod_name}__staging_{ts}")
         except Exception:
@@ -4550,7 +4556,11 @@ def _run_transport(transport: str) -> None:
     import uvicorn
 
     app_factory = _http_app_factory(transport)
-    app = app_factory()
+    # Build the app with the configured host. MCP 2.0.0's factories default to
+    # host="127.0.0.1", which turns on a localhost-only DNS-rebinding allowlist;
+    # without this, a non-loopback server_host is served by uvicorn but the app
+    # rejects the configured host with 421 before auth runs (GH #174).
+    app = app_factory(host=config.server_host)
     guarded = BearerAuthMiddleware(app, token)
     print(
         f"[SECURITY] Bearer auth enforced on {transport} transport ({config.server_host}:{config.server_port})",

{knowledge_rag-4.8.3 → knowledge_rag-4.8.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "knowledge-rag"
-version = "4.8.3"
+version = "4.8.4"
 description = "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 13 MCP Tools + 20 Format Parsers. Zero external servers."
 readme = "README.md"
 license = {text = "MIT"}