knowledge-rag 3.3.1__tar.gz → 3.4.0__tar.gz

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/.gitignore

@@ -28,6 +28,7 @@ documents/**/*.csv
 
 # FastEmbed model cache
 .cache/
+models_cache/
 *.onnx
 
 # Local scripts (not part of distribution)

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: knowledge-rag
-Version: 3.3.1
+Version: 3.4.0
 Summary: Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 12 MCP Tools. Zero external servers.
 Project-URL: Homepage, https://github.com/lyonzin/knowledge-rag
 Project-URL: Repository, https://github.com/lyonzin/knowledge-rag
@@ -38,7 +38,7 @@ Description-Content-Type: text/markdown
 
 <div align="center">
 
-![Version](https://img.shields.io/badge/version-3.3.1-blue.svg)
+![Version](https://img.shields.io/badge/version-3.4.0-blue.svg)
 ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12-green.svg)
 ![License](https://img.shields.io/badge/license-MIT-yellow.svg)
 ![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg)
@@ -61,7 +61,7 @@ Your documents become instantly searchable inside Claude Code — with reranking
 
 **12 MCP Tools** | **Hybrid Search + Cross-Encoder Reranking** | **Markdown-Aware Chunking** | **100% Local, Zero Cloud**
 
-[What's New](#whats-new-in-v330) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
+[What's New](#whats-new-in-v340) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
 
 </div>
 
@@ -108,6 +108,40 @@ After the initial rebuild, startup and queries are faster than v2.x because ther
 
 ---
 
+## What's New in v3.4.0
+
+### Persistent Model Cache
+Embedding models (~250MB) are now stored in `models_cache/` instead of `/tmp`. No more re-downloading after system reboots on Linux. Configurable via `paths.models_cache_dir` in config.yaml.
+
+### Exclude Patterns
+Skip files and directories during indexing with glob patterns. Configure under `documents.exclude_patterns` in config.yaml:
+```yaml
+documents:
+  exclude_patterns:
+    - "node_modules"
+    - ".venv"
+    - "__pycache__"
+    - ".git"
+    - "*.tmp"
+```
+Patterns match against both full relative paths and individual path components — `"node_modules"` excludes it at any depth.
+
+### Jupyter Notebook Support (.ipynb)
+Native parser extracts only markdown and code cell sources — no outputs, no execution counts, no base64-encoded images. Clean content for high-quality retrieval. Added to default supported formats.
+
+### MCP Protocol Stability
+`stdout` is redirected to `stderr` before the MCP server starts. Prevents random library `print()` calls from corrupting the JSON-RPC stream — fixes "Failed to connect" errors in VS Code and IDE integrations.
+
+### File Watcher Resilience (Linux)
+Server no longer crashes when system inotify limits are reached. Falls back gracefully to manual indexing with a clear warning message.
+
+### MetaTrader Support (.mq4, .mqh)
+MQL4/MQL5 source files can now be indexed as code. Not enabled by default — add `.mq4` and `.mqh` to your `supported_formats` list to activate.
+
+*Community credit: Features inspired by ideas from [@Hohlas](https://github.com/Hohlas) in [PR #18](https://github.com/lyonzin/knowledge-rag/pull/18).*
+
+---
+
 ## What's New in v3.3.0
 
 ### YAML Configuration System
@@ -966,6 +1000,7 @@ Pre-built configurations for common use cases. Each preset is a complete `config
 |-------|---------|-------------|
 | `paths.documents_dir` | `./documents` | Root folder scanned recursively for documents |
 | `paths.data_dir` | `./data` | Internal storage for ChromaDB and index metadata |
+| `paths.models_cache_dir` | `./models_cache` | Persistent cache for embedding models (~250MB). Prevents re-download after reboots |
 
 Relative paths resolve from the project root. Absolute paths work too. The `KNOWLEDGE_RAG_DIR` environment variable overrides the project root.
 
@@ -973,7 +1008,8 @@ Relative paths resolve from the project root. Absolute paths work too. The `KNOW
 
 | Field | Default | Description |
 |-------|---------|-------------|
-| `documents.supported_formats` | .md .txt .pdf .py .json .docx .xlsx .pptx .csv | File extensions to index |
+| `documents.supported_formats` | .md .txt .pdf .py .json .docx .xlsx .pptx .csv .ipynb | File extensions to index |
+| `documents.exclude_patterns` | `[]` (empty) | Glob patterns for files/dirs to skip during indexing |
 | `documents.chunking.chunk_size` | 1000 | Max characters per chunk |
 | `documents.chunking.chunk_overlap` | 200 | Characters shared between consecutive chunks |
 
@@ -1090,7 +1126,8 @@ knowledge-rag/
 ├── data/
 │   ├── chroma_db/           # ChromaDB vector database
 │   └── index_metadata.json  # Incremental indexing state
-├── tests/                   # Test suite (55 tests)
+├── models_cache/            # Persistent embedding model cache
+├── tests/                   # Test suite (70+ tests)
 ├── venv/                    # Python virtual environment
 ├── requirements.txt         # Python dependencies
 ├── LICENSE                  # MIT License
@@ -1163,7 +1200,7 @@ pip install rank-bm25
 
 ### "ModuleNotFoundError: No module named 'mcp_server'"
 
-This occurs when Claude Code doesn't set the working directory correctly. Use the `cmd /c "cd /d ... && python"` wrapper in your MCP config (see [Installation](#configure-mcp-for-claude-code)).
+This occurs when Claude Code doesn't set the working directory correctly. Use the `cmd /c "cd /d ... && python"` wrapper in your MCP config (see [Installation](#installation)).
 
 ### Dimension mismatch after upgrade
 
@@ -1195,6 +1232,27 @@ With ~200 documents, expect ~300-500MB RAM. The embedding model (~50MB) and rera
 
 ## Changelog
 
+### v3.4.0 (2026-04-16)
+
+- **NEW**: `models_cache_dir` — persistent embedding model cache, prevents re-download after reboots
+- **NEW**: `exclude_patterns` — glob-based file/directory exclusion during indexing (under `documents:` section)
+- **NEW**: Jupyter Notebook (.ipynb) parser — extracts markdown and code cell sources only, ignores outputs and base64
+- **NEW**: MCP stdout protection — redirects stdout to stderr before server start, fixes "Failed to connect" errors
+- **NEW**: File watcher resilience — graceful fallback when Linux inotify limits are reached
+- **NEW**: MetaTrader (.mq4, .mqh) support — opt-in code parsing for MQL4/MQL5 files
+- **NEW**: 23 new tests (exclude patterns, ipynb parser, stdout protection)
+- **IMPROVED**: All 4 presets updated with new config fields
+- Community credit: [@Hohlas](https://github.com/Hohlas) ([PR #18](https://github.com/lyonzin/knowledge-rag/pull/18))
+
+### v3.3.2 (2026-04-06)
+
+- **FIX**: Full type validation on all YAML config values — wrong types warn and fall back to defaults
+- **FIX**: Bounds validation for chunk_size, chunk_overlap, default_results, max_results, embedding_dim
+- **FIX**: `keyword_routes` with string values instead of lists detected and removed with warning
+- **FIX**: `reranker_enabled: "yes"` (string) corrected to boolean with warning
+- **FIX**: Synced version strings across all source files
+- **FIX**: Broken README anchor, duplicate keyword, error handling in `knowledge-rag init`
+
 ### v3.3.1 (2026-04-06)
 
 - **FIX**: YAML null values (`category_mappings:` without value) no longer crash the server — falls back to defaults

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/README.md

@@ -2,7 +2,7 @@
 
 <div align="center">
 
-![Version](https://img.shields.io/badge/version-3.3.1-blue.svg)
+![Version](https://img.shields.io/badge/version-3.4.0-blue.svg)
 ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12-green.svg)
 ![License](https://img.shields.io/badge/license-MIT-yellow.svg)
 ![Platform](https://img.shields.io/badge/platform-Windows%20%7C%20Linux%20%7C%20macOS-lightgrey.svg)
@@ -25,7 +25,7 @@ Your documents become instantly searchable inside Claude Code — with reranking
 
 **12 MCP Tools** | **Hybrid Search + Cross-Encoder Reranking** | **Markdown-Aware Chunking** | **100% Local, Zero Cloud**
 
-[What's New](#whats-new-in-v330) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
+[What's New](#whats-new-in-v340) | [Installation](#installation) | [Configuration](#configuration) | [API Reference](#api-reference) | [Architecture](#architecture)
 
 </div>
 
@@ -72,6 +72,40 @@ After the initial rebuild, startup and queries are faster than v2.x because ther
 
 ---
 
+## What's New in v3.4.0
+
+### Persistent Model Cache
+Embedding models (~250MB) are now stored in `models_cache/` instead of `/tmp`. No more re-downloading after system reboots on Linux. Configurable via `paths.models_cache_dir` in config.yaml.
+
+### Exclude Patterns
+Skip files and directories during indexing with glob patterns. Configure under `documents.exclude_patterns` in config.yaml:
+```yaml
+documents:
+  exclude_patterns:
+    - "node_modules"
+    - ".venv"
+    - "__pycache__"
+    - ".git"
+    - "*.tmp"
+```
+Patterns match against both full relative paths and individual path components — `"node_modules"` excludes it at any depth.
+
+### Jupyter Notebook Support (.ipynb)
+Native parser extracts only markdown and code cell sources — no outputs, no execution counts, no base64-encoded images. Clean content for high-quality retrieval. Added to default supported formats.
+
+### MCP Protocol Stability
+`stdout` is redirected to `stderr` before the MCP server starts. Prevents random library `print()` calls from corrupting the JSON-RPC stream — fixes "Failed to connect" errors in VS Code and IDE integrations.
+
+### File Watcher Resilience (Linux)
+Server no longer crashes when system inotify limits are reached. Falls back gracefully to manual indexing with a clear warning message.
+
+### MetaTrader Support (.mq4, .mqh)
+MQL4/MQL5 source files can now be indexed as code. Not enabled by default — add `.mq4` and `.mqh` to your `supported_formats` list to activate.
+
+*Community credit: Features inspired by ideas from [@Hohlas](https://github.com/Hohlas) in [PR #18](https://github.com/lyonzin/knowledge-rag/pull/18).*
+
+---
+
 ## What's New in v3.3.0
 
 ### YAML Configuration System
@@ -930,6 +964,7 @@ Pre-built configurations for common use cases. Each preset is a complete `config
 |-------|---------|-------------|
 | `paths.documents_dir` | `./documents` | Root folder scanned recursively for documents |
 | `paths.data_dir` | `./data` | Internal storage for ChromaDB and index metadata |
+| `paths.models_cache_dir` | `./models_cache` | Persistent cache for embedding models (~250MB). Prevents re-download after reboots |
 
 Relative paths resolve from the project root. Absolute paths work too. The `KNOWLEDGE_RAG_DIR` environment variable overrides the project root.
 
@@ -937,7 +972,8 @@ Relative paths resolve from the project root. Absolute paths work too. The `KNOW
 
 | Field | Default | Description |
 |-------|---------|-------------|
-| `documents.supported_formats` | .md .txt .pdf .py .json .docx .xlsx .pptx .csv | File extensions to index |
+| `documents.supported_formats` | .md .txt .pdf .py .json .docx .xlsx .pptx .csv .ipynb | File extensions to index |
+| `documents.exclude_patterns` | `[]` (empty) | Glob patterns for files/dirs to skip during indexing |
 | `documents.chunking.chunk_size` | 1000 | Max characters per chunk |
 | `documents.chunking.chunk_overlap` | 200 | Characters shared between consecutive chunks |
 
@@ -1054,7 +1090,8 @@ knowledge-rag/
 ├── data/
 │   ├── chroma_db/           # ChromaDB vector database
 │   └── index_metadata.json  # Incremental indexing state
-├── tests/                   # Test suite (55 tests)
+├── models_cache/            # Persistent embedding model cache
+├── tests/                   # Test suite (70+ tests)
 ├── venv/                    # Python virtual environment
 ├── requirements.txt         # Python dependencies
 ├── LICENSE                  # MIT License
@@ -1127,7 +1164,7 @@ pip install rank-bm25
 
 ### "ModuleNotFoundError: No module named 'mcp_server'"
 
-This occurs when Claude Code doesn't set the working directory correctly. Use the `cmd /c "cd /d ... && python"` wrapper in your MCP config (see [Installation](#configure-mcp-for-claude-code)).
+This occurs when Claude Code doesn't set the working directory correctly. Use the `cmd /c "cd /d ... && python"` wrapper in your MCP config (see [Installation](#installation)).
 
 ### Dimension mismatch after upgrade
 
@@ -1159,6 +1196,27 @@ With ~200 documents, expect ~300-500MB RAM. The embedding model (~50MB) and rera
 
 ## Changelog
 
+### v3.4.0 (2026-04-16)
+
+- **NEW**: `models_cache_dir` — persistent embedding model cache, prevents re-download after reboots
+- **NEW**: `exclude_patterns` — glob-based file/directory exclusion during indexing (under `documents:` section)
+- **NEW**: Jupyter Notebook (.ipynb) parser — extracts markdown and code cell sources only, ignores outputs and base64
+- **NEW**: MCP stdout protection — redirects stdout to stderr before server start, fixes "Failed to connect" errors
+- **NEW**: File watcher resilience — graceful fallback when Linux inotify limits are reached
+- **NEW**: MetaTrader (.mq4, .mqh) support — opt-in code parsing for MQL4/MQL5 files
+- **NEW**: 23 new tests (exclude patterns, ipynb parser, stdout protection)
+- **IMPROVED**: All 4 presets updated with new config fields
+- Community credit: [@Hohlas](https://github.com/Hohlas) ([PR #18](https://github.com/lyonzin/knowledge-rag/pull/18))
+
+### v3.3.2 (2026-04-06)
+
+- **FIX**: Full type validation on all YAML config values — wrong types warn and fall back to defaults
+- **FIX**: Bounds validation for chunk_size, chunk_overlap, default_results, max_results, embedding_dim
+- **FIX**: `keyword_routes` with string values instead of lists detected and removed with warning
+- **FIX**: `reranker_enabled: "yes"` (string) corrected to boolean with warning
+- **FIX**: Synced version strings across all source files
+- **FIX**: Broken README anchor, duplicate keyword, error handling in `knowledge-rag init`
+
 ### v3.3.1 (2026-04-06)
 
 - **FIX**: YAML null values (`category_mappings:` without value) no longer crash the server — falls back to defaults

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/config.example.yaml

@@ -39,6 +39,11 @@ paths:
   # You generally don't need to touch this.
   data_dir: "./data"
 
+  # Persistent cache for embedding models (~250MB).
+  # Prevents re-downloading after reboots (especially on Linux where /tmp is cleared).
+  # Default: ./models_cache (relative to project root)
+  models_cache_dir: "./models_cache"
+
 
 # ============================================================================
 # DOCUMENTS
@@ -58,6 +63,25 @@ documents:
     # - .csv     # CSV data files
     # - .py      # Python source code
     # - .json    # JSON files
+    # - .ipynb   # Jupyter Notebooks (extracts markdown + code cells)
+    # - .mq4     # MetaTrader MQL4 source (opt-in, add to enable)
+    # - .mqh     # MetaTrader MQL4/5 headers (opt-in, add to enable)
+
+  # Exclude patterns — skip files/directories matching these patterns.
+  # Uses fnmatch glob syntax. Patterns match against relative paths AND
+  # individual path components (so "node_modules" matches at any depth).
+  #
+  # Examples:
+  #   exclude_patterns:
+  #     - "node_modules"     # Skip node_modules at any depth
+  #     - ".git"             # Skip .git directories
+  #     - "__pycache__"      # Skip Python cache
+  #     - ".venv"            # Skip virtual environments
+  #     - "*.tmp"            # Skip .tmp files anywhere
+  #     - "drafts/*"         # Skip everything in top-level drafts/
+  #
+  # Default: [] (nothing excluded — all supported files are indexed)
+  exclude_patterns: []
 
   # How documents are split into searchable chunks.
   #

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/mcp_server/__init__.py

@@ -1,6 +1,6 @@
 """Knowledge RAG MCP Server - Local Retrieval-Augmented Generation System"""
 
-__version__ = "3.2.3"
+__version__ = "3.4.0"
 __author__ = "Ailton Rocha (Lyon.)"
 
 from .config import Config

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/mcp_server/config.py

@@ -1,4 +1,4 @@
-"""Configuration for Knowledge RAG System v3.1 — YAML-configurable"""
+"""Configuration for Knowledge RAG System v3.4.0 — YAML-configurable"""
 
 import os
 from dataclasses import dataclass, field
@@ -15,7 +15,7 @@ import yaml
 _source_dir = Path(__file__).parent.parent
 
 
-_SUPPORTED_SUFFIXES = frozenset([".md", ".txt", ".pdf", ".py", ".json", ".docx", ".xlsx", ".pptx", ".csv"])
+_SUPPORTED_SUFFIXES = frozenset([".md", ".txt", ".pdf", ".py", ".json", ".docx", ".xlsx", ".pptx", ".csv", ".ipynb"])
 
 
 def _has_documents(path: Path) -> bool:
@@ -73,7 +73,19 @@ def _get(section: str, key: str, default):
     if not isinstance(s, dict):
         return default
     val = s.get(key)
-    return val if val is not None else default
+    if val is None:
+        return default
+    # Skip type check when default is None (caller handles validation)
+    if default is None:
+        return val
+    # YAML parses "yes"/"no" as bool, but explicit string "yes" stays str
+    if not isinstance(val, type(default)):
+        print(
+            f"[WARN] config.yaml: {section}.{key} has wrong type "
+            f"(expected {type(default).__name__}, got {type(val).__name__}), using default"
+        )
+        return default
+    return val
 
 
 def _get_top(key: str, default):
@@ -81,7 +93,8 @@ def _get_top(key: str, default):
     val = _yaml.get(key)
     if val is None:
         return default
-    if not isinstance(val, type(default)):
+    if not isinstance(val, dict):
+        print(f"[WARN] config.yaml: {key} has wrong type (expected dict, got {type(val).__name__}), using default")
         return default
     return val
 
@@ -155,7 +168,6 @@ _DEFAULT_KEYWORD_ROUTES = {
         "deserialization",
         "ysoserial",
         "upload bypass",
-        "reverse shell",
         "web shell",
         "hash cracking",
         "hashcat",
@@ -345,6 +357,9 @@ class Config:
     documents_dir: Path = field(
         default_factory=lambda: _resolve_path(_get("paths", "documents_dir", None), BASE_DIR / "documents")
     )
+    models_cache_dir: Path = field(
+        default_factory=lambda: _resolve_path(_get("paths", "models_cache_dir", None), BASE_DIR / "models_cache")
+    )
 
     # Chunking
     chunk_size: int = field(
@@ -407,10 +422,15 @@ class Config:
     # Supported formats
     supported_formats: List[str] = field(
         default_factory=lambda: _get(
-            "documents", "supported_formats", [".md", ".txt", ".pdf", ".py", ".json", ".docx", ".xlsx", ".pptx", ".csv"]
+            "documents",
+            "supported_formats",
+            [".md", ".txt", ".pdf", ".py", ".json", ".docx", ".xlsx", ".pptx", ".csv", ".ipynb"],
         )
     )
 
+    # Exclude patterns for directory traversal
+    exclude_patterns: List[str] = field(default_factory=lambda: _get("documents", "exclude_patterns", []))
+
     # Category mappings
     category_mappings: Dict[str, str] = field(
         default_factory=lambda: _get_top("category_mappings", _DEFAULT_CATEGORY_MAPPINGS)
@@ -431,10 +451,52 @@ class Config:
     max_results: int = field(default_factory=lambda: _get("search", "max_results", 20))
 
     def __post_init__(self):
-        """Ensure directories exist"""
+        """Validate config values and ensure directories exist."""
+        # Bounds validation
+        if not isinstance(self.chunk_size, int) or self.chunk_size < 100:
+            print(f"[WARN] chunk_size={self.chunk_size} invalid, using 1000")
+            self.chunk_size = 1000
+        if not isinstance(self.chunk_overlap, int) or self.chunk_overlap < 0:
+            print(f"[WARN] chunk_overlap={self.chunk_overlap} invalid, using 200")
+            self.chunk_overlap = 200
+        if self.chunk_overlap >= self.chunk_size:
+            print(
+                f"[WARN] chunk_overlap ({self.chunk_overlap}) >= chunk_size ({self.chunk_size}), using {self.chunk_size // 5}"
+            )
+            self.chunk_overlap = self.chunk_size // 5
+        if not isinstance(self.default_results, int) or self.default_results < 1:
+            self.default_results = 5
+        if not isinstance(self.max_results, int) or self.max_results < 1:
+            self.max_results = 20
+        if not isinstance(self.embedding_dim, int) or self.embedding_dim < 1:
+            self.embedding_dim = 384
+        if not isinstance(self.reranker_enabled, bool):
+            print(f"[WARN] reranker_enabled={self.reranker_enabled!r} invalid, using True")
+            self.reranker_enabled = True
+        if not isinstance(self.reranker_top_k_multiplier, int) or self.reranker_top_k_multiplier < 1:
+            self.reranker_top_k_multiplier = 3
+        if not isinstance(self.supported_formats, list) or not self.supported_formats:
+            print("[WARN] supported_formats is empty or invalid, using defaults")
+            self.supported_formats = [".md", ".txt", ".pdf", ".py", ".json", ".docx", ".xlsx", ".pptx", ".csv"]
+
+        # Validate exclude_patterns is a list of strings
+        if not isinstance(self.exclude_patterns, list):
+            print(f"[WARN] exclude_patterns={self.exclude_patterns!r} invalid, using []")
+            self.exclude_patterns = []
+        else:
+            self.exclude_patterns = [p for p in self.exclude_patterns if isinstance(p, str)]
+
+        # Validate keyword_routes values are lists (not strings)
+        for cat, keywords in list(self.keyword_routes.items()):
+            if not isinstance(keywords, list):
+                print(f"[WARN] keyword_routes.{cat} is not a list, removing")
+                del self.keyword_routes[cat]
+
+        # Ensure directories exist
         self.data_dir.mkdir(parents=True, exist_ok=True)
         self.chroma_dir.mkdir(parents=True, exist_ok=True)
         self.documents_dir.mkdir(parents=True, exist_ok=True)
+        self.models_cache_dir.mkdir(parents=True, exist_ok=True)
 
 
 # Global config instance

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/mcp_server/ingestion.py

@@ -1,9 +1,10 @@
 """Document Ingestion System for Knowledge RAG
 
 Multi-format document parsing, chunking, and metadata extraction.
-Supports: MD, PDF, TXT, PY, JSON, DOCX, XLSX, PPTX, CSV
+Supports: MD, PDF, TXT, PY, JSON, DOCX, XLSX, PPTX, CSV, IPYNB, MQH, MQ4
 """
 
+import fnmatch
 import hashlib
 import json
 import os
@@ -103,6 +104,9 @@ class DocumentParser:
             ".xlsx": self._parse_xlsx,
             ".pptx": self._parse_pptx,
             ".csv": self._parse_csv,
+            ".ipynb": self._parse_ipynb,
+            ".mqh": self._parse_code,
+            ".mq4": self._parse_code,
         }
 
     def parse_file(self, filepath: Path) -> Optional[Document]:
@@ -151,12 +155,41 @@ class DocumentParser:
 
         return doc
 
+    @staticmethod
+    def _should_exclude(path: Path, base_dir: Path, patterns: List[str]) -> bool:
+        """Check if a path matches any exclude pattern.
+
+        Uses fnmatch on the relative path (forward-slash normalized) and
+        also checks each path component individually for simple name patterns.
+        """
+        if not patterns:
+            return False
+
+        try:
+            rel = path.relative_to(base_dir)
+        except ValueError:
+            rel = path
+
+        rel_str = str(rel).replace("\\", "/")
+
+        for pattern in patterns:
+            # Full relative path match (e.g., "docs/drafts/*.tmp")
+            if fnmatch.fnmatch(rel_str, pattern):
+                return True
+            # Check each component (e.g., "node_modules" matches any/node_modules/deep)
+            for part in rel.parts:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+
+        return False
+
     def parse_directory(self, directory: Path = None) -> List[Document]:
         """Parse all supported files in a directory recursively (follows symlinks)."""
         directory = Path(directory) if directory else config.documents_dir
         documents = []
         seen_dirs = set()
         supported = set(config.supported_formats)
+        exclude = config.exclude_patterns
 
         for root, dirs, files in os.walk(directory, followlinks=True):
             real_root = os.path.realpath(root)
@@ -165,10 +198,16 @@ class DocumentParser:
                 continue
             seen_dirs.add(real_root)
 
+            # Filter out excluded directories in-place (prevents os.walk from descending)
+            if exclude:
+                dirs[:] = [d for d in dirs if not self._should_exclude(Path(root) / d, directory, exclude)]
+
             for fname in files:
                 filepath = Path(root) / fname
                 if filepath.suffix.lower() not in supported:
                     continue
+                if exclude and self._should_exclude(filepath, directory, exclude):
+                    continue
                 try:
                     doc = self.parse_file(filepath)
                     if doc:
@@ -437,6 +476,59 @@ class DocumentParser:
         content = "\n".join(parts)
         return content, metadata
 
+    def _parse_ipynb(self, filepath: Path) -> tuple[str, Dict]:
+        """Parse Jupyter Notebook, extracting only markdown and code cell sources.
+
+        Ignores outputs, execution counts, cell metadata, and base64 images.
+        """
+        raw = filepath.read_text(encoding="utf-8", errors="ignore")
+        metadata = {
+            "type": "jupyter_notebook",
+            "title": filepath.stem,
+            "file_size": filepath.stat().st_size,
+            "modified": datetime.fromtimestamp(filepath.stat().st_mtime).isoformat(),
+        }
+
+        try:
+            nb = json.loads(raw)
+        except json.JSONDecodeError:
+            metadata["is_valid_json"] = False
+            return raw, metadata
+
+        metadata["is_valid_json"] = True
+        metadata["nbformat"] = nb.get("nbformat", 0)
+        kernel = nb.get("metadata", {}).get("kernelspec", {})
+        metadata["kernel"] = kernel.get("display_name", kernel.get("name", "unknown"))
+
+        cells = nb.get("cells", [])
+        metadata["cells"] = len(cells)
+        code_cells = 0
+        markdown_cells = 0
+
+        parts = []
+        for cell in cells:
+            cell_type = cell.get("cell_type", "")
+            source = cell.get("source", "")
+
+            if isinstance(source, list):
+                source = "".join(source)
+
+            if not source or not source.strip():
+                continue
+
+            if cell_type == "markdown":
+                parts.append(source)
+                markdown_cells += 1
+            elif cell_type == "code":
+                parts.append(f"```python\n{source}\n```")
+                code_cells += 1
+
+        metadata["code_cells"] = code_cells
+        metadata["markdown_cells"] = markdown_cells
+
+        content = "\n\n".join(parts)
+        return content, metadata
+
     # =========================================================================
     # Chunking
     # =========================================================================

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/mcp_server/server.py

@@ -19,8 +19,8 @@ Features:
     - CRUD operations via MCP tools (add, update, remove docs)
 
 Autor:   Lyon (Ailton Rocha)
-Versao:  3.0.0
-Data:    2026-03-19
+Versao:  3.4.0
+Data:    2026-04-16
 """
 
 import hashlib
@@ -141,7 +141,7 @@ class FastEmbedEmbeddings:
         self.model_name = model or config.embedding_model
         self._dim = config.embedding_dim
         print(f"[INFO] Loading embedding model: {self.model_name} ({self._dim}D)...")
-        self._model = TextEmbedding(model_name=self.model_name)
+        self._model = TextEmbedding(model_name=self.model_name, cache_dir=str(config.models_cache_dir))
         print("[INFO] Embedding model loaded successfully")
 
     def __call__(self, input: List[str]) -> List[List[float]]:
@@ -1836,32 +1836,37 @@ def _handle_init():
 
     cwd = Path.cwd()
 
-    # Copy config.example.yaml
-    src = data_dir / "config.example.yaml"
-    if src.exists():
-        dst = cwd / "config.example.yaml"
-        shutil.copy2(src, dst)
-        print(f"[OK] {dst}")
-
-    # Copy presets
-    presets_dir = cwd / "presets"
-    presets_dir.mkdir(exist_ok=True)
-    for f in data_dir.glob("*.yaml"):
-        if f.name == "config.example.yaml":
-            continue
-        dst = presets_dir / f.name
-        shutil.copy2(f, dst)
-        print(f"[OK] {dst}")
-
-    # Create documents dir
-    docs_dir = cwd / "documents"
-    docs_dir.mkdir(exist_ok=True)
-    print(f"[OK] {docs_dir}/")
-
-    print("\nDone. Quick start:")
-    print("  cp presets/general.yaml config.yaml     # or cybersecurity, developer, research")
-    print("  # Add your documents to documents/")
-    print("  # Restart Claude Code")
+    try:
+        # Copy config.example.yaml
+        src = data_dir / "config.example.yaml"
+        if src.exists():
+            dst = cwd / "config.example.yaml"
+            shutil.copy2(src, dst)
+            print(f"[OK] {dst}")
+
+        # Copy presets
+        presets_dir = cwd / "presets"
+        presets_dir.mkdir(exist_ok=True)
+        for f in data_dir.glob("*.yaml"):
+            if f.name == "config.example.yaml":
+                continue
+            dst = presets_dir / f.name
+            shutil.copy2(f, dst)
+            print(f"[OK] {dst}")
+
+        # Create documents dir
+        docs_dir = cwd / "documents"
+        docs_dir.mkdir(exist_ok=True)
+        print(f"[OK] {docs_dir}/")
+
+        print("\nDone. Quick start:")
+        print("  cp presets/general.yaml config.yaml     # or cybersecurity, developer, research")
+        print("  # Add your documents to documents/")
+        print("  # Restart Claude Code")
+    except PermissionError:
+        print("[ERROR] Permission denied. Run from a writable directory.")
+    except OSError as e:
+        print(f"[ERROR] Failed to write files: {e}")
 
 
 def main():
@@ -1894,12 +1899,21 @@ def main():
         print(f"[INFO] Indexed {stats['indexed']} documents with {stats['chunks_added']} chunks")
 
     # Start file watcher for auto-reindex on document changes
-    watcher = DocumentWatcher(get_orchestrator, debounce_seconds=5.0)
-    observer = Observer()
-    observer.schedule(watcher, str(config.documents_dir), recursive=True)
-    observer.daemon = True
-    observer.start()
-    print(f"[WATCHER] Monitoring {config.documents_dir} for changes")
+    try:
+        watcher = DocumentWatcher(get_orchestrator, debounce_seconds=5.0)
+        observer = Observer()
+        observer.schedule(watcher, str(config.documents_dir), recursive=True)
+        observer.daemon = True
+        observer.start()
+        print(f"[WATCHER] Monitoring {config.documents_dir} for changes")
+    except Exception as e:
+        print(f"[WARN] Failed to start file watcher: {e}")
+        print("[WARN] Auto-reindexing disabled. Use reindex_documents tool manually.")
+
+    # Redirect stdout to stderr to protect MCP stdio JSON-RPC stream.
+    # Libraries (fastembed, chromadb, watchdog) may print() at runtime,
+    # which would corrupt the protocol. This ONLY affects the server process.
+    sys.stdout = sys.stderr
 
     mcp.run()
 

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/presets/cybersecurity.yaml

@@ -22,6 +22,7 @@
 paths:
   documents_dir: "./documents"
   data_dir: "./data"
+  models_cache_dir: "./models_cache"
 
 
 # ============================================================================
@@ -39,6 +40,9 @@ documents:
     - .xlsx
     - .pptx
     - .csv
+    - .ipynb    # Jupyter Notebooks (exploit dev, research)
+
+  exclude_patterns: []
 
   chunking:
     chunk_size: 1000

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/presets/developer.yaml

@@ -17,6 +17,7 @@
 paths:
   documents_dir: "./documents"
   data_dir: "./data"
+  models_cache_dir: "./models_cache"
 
 
 # ============================================================================
@@ -32,6 +33,16 @@ documents:
     - .py        # Python source code
     - .json      # API schemas, configs
     - .csv       # Data files, logs
+    - .ipynb     # Jupyter Notebooks
+
+  exclude_patterns:
+    - "node_modules"
+    - ".venv"
+    - "__pycache__"
+    - ".git"
+    - "dist"
+    - "build"
+    - ".next"
 
   chunking:
     chunk_size: 1200      # Slightly larger for code + prose docs

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/presets/general.yaml

@@ -26,6 +26,7 @@
 paths:
   documents_dir: "./documents"
   data_dir: "./data"
+  models_cache_dir: "./models_cache"
 
 
 # ============================================================================
@@ -39,6 +40,8 @@ documents:
     - .pdf
     - .docx
 
+  exclude_patterns: []
+
   chunking:
     chunk_size: 1000
     chunk_overlap: 200

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/presets/research.yaml

@@ -17,6 +17,7 @@
 paths:
   documents_dir: "./documents"
   data_dir: "./data"
+  models_cache_dir: "./models_cache"
 
 
 # ============================================================================
@@ -32,6 +33,9 @@ documents:
     - .xlsx      # Data tables, survey results
     - .pptx      # Presentations, lecture slides
     - .csv       # Experiment data, datasets
+    - .ipynb     # Jupyter Notebooks (analysis, experiments)
+
+  exclude_patterns: []
 
   chunking:
     chunk_size: 1500      # Larger chunks — academic text needs more context

{knowledge_rag-3.3.1 → knowledge_rag-3.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "knowledge-rag"
-version = "3.3.1"
+version = "3.4.0"
 description = "Local RAG System for Claude Code — Hybrid search + Cross-encoder Reranking + 12 MCP Tools. Zero external servers."
 readme = "README.md"
 license = {text = "MIT"}