document-rag-mcp 0.1.0__tar.gz

document_rag_mcp-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,21 @@
+name: docs
+on:
+  push:
+    branches:
+      - master
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Install dependencies
+        run: uv sync --group docs
+      - name: Build and deploy docs
+        run: uv run mkdocs gh-deploy --force

document_rag_mcp-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,59 @@
+name: release
+
+on:
+  push:
+    tags:
+      - 'v*' # Trigger on tags like v1.0.0, v0.1.0, etc.
+
+jobs:
+  build:
+    name: Build Python distribution 📦
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+      - name: Build distribution 📦
+        run: uv build
+      - name: Upload distribution 📦
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  pypi-publish:
+    name: Publish Python distribution 📦 to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/document-rag-mcp
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+      contents: read
+    steps:
+      - name: Download distribution 📦
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download distribution 📦
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true

document_rag_mcp-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Run Ruff Lint and Format Checks
+        run: uv run ruff check src/ tests/
+
+      - name: Run Tests with Coverage
+        run: uv run pytest --cov=document_rag_mcp --cov-report=xml -v

document_rag_mcp-0.1.0/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Pyinstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nosenv/
+.pytest_cache/
+.pickle_cache/
+.cache/
+.coverage
+.coverage.*
+.xml
+cov.xml
+.hypothesis/
+.htmlcov/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or app, you might want to share your .python-version.
+#   For an executable, it's typically fine to ignore.
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if you choose to keep it in git:
+#Pipfile.lock
+
+# poetry
+#   Similarly, in case of poetry, it's recommended to include poetry.lock in version control.
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is NOT recommended to include pdm.lock in version control.
+#pdm.lock
+.pdm-plugins/
+
+# celry beat schedule file
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# Local databases and runtime state data directories
+/data/
+*.db
+*.db-journal
+chroma/

document_rag_mcp-0.1.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: document-rag-mcp
+Version: 0.1.0
+Summary: RAG MCP Server — watches folders, chunks documents, serves semantic search via MCP
+License: AGPL-3.0-or-later
+Requires-Python: >=3.11
+Requires-Dist: chonkie[semantic]>=1.0
+Requires-Dist: chromadb>=1.5
+Requires-Dist: click>=8.0
+Requires-Dist: mcp[cli]<2,>=1.27
+Requires-Dist: openai>=1.80
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pymupdf>=1.25
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: watchfiles>=1.0
+Description-Content-Type: text/markdown
+
+# document-rag-mcp
+
+A RAG MCP (Model Context Protocol) server. It recursively scans and watches configured directories for `.txt`, `.md`, and `.pdf` files, semantically chunks them, computes embeddings using an OpenAI-compatible API, and stores them in an embedded ChromaDB instance.
+
+It exposes tools to search documents and retrieve original content/metadata over the MCP protocol.

document_rag_mcp-0.1.0/README.md

@@ -0,0 +1,5 @@
+# document-rag-mcp
+
+A RAG MCP (Model Context Protocol) server. It recursively scans and watches configured directories for `.txt`, `.md`, and `.pdf` files, semantically chunks them, computes embeddings using an OpenAI-compatible API, and stores them in an embedded ChromaDB instance.
+
+It exposes tools to search documents and retrieve original content/metadata over the MCP protocol.

document_rag_mcp-0.1.0/config.example.yaml

@@ -0,0 +1,53 @@
+# document-rag-mcp configuration example
+
+# Define your document collections.
+# Each collection groups one or more folders and scans them recursively.
+collections:
+  - name: "project-docs"
+    paths:
+      - "/absolute/path/to/my-project/docs"
+      - "/absolute/path/to/another/folder"
+    file_patterns:
+      - "*.txt"
+      - "*.md"
+      - "*.pdf"
+
+  - name: "research-papers"
+    paths:
+      - "/absolute/path/to/papers"
+    file_patterns:
+      - "*.pdf"
+
+# Remote embedding model config via an OpenAI-compatible endpoint.
+# Compatible with lemonade, OpenRouter, Ollama, etc.
+embedding:
+  base_url: "http://localhost:8080/v1"   # Default local lemonade endpoint
+  api_key: "unused"                     # Use "unused" or actual key
+  model: "embed-gemma-300m-FLM"         # Default embedding model
+  dimensions: 768                       # Dimensionality of the vectors
+  batch_size: 32                        # Batch size for embedding requests
+
+# Local semantic chunking configurations.
+chunking:
+  max_chunk_size: 512                   # Maximum tokens per chunk
+  similarity_threshold: 0.5             # Topic shift threshold for semantic chunking
+  local_model: "all-MiniLM-L6-v2"       # Model used for local semantic boundary detection
+                                        # Note: Falls back to TokenChunker if sentence-transformers is not installed.
+
+# Storage paths for metadata and vector storage.
+storage:
+  data_dir: "./data"                    # Path where SQLite and ChromaDB data will be saved
+
+# Optional Vision LLM config for scanned/un-OCRed PDFs.
+# Set enabled to true to run page-to-image extraction via multimodal completions.
+vision:
+  enabled: false
+  base_url: "http://localhost:8080/v1"   # Vision endpoint URL
+  api_key: "unused"                     # Vision API Key
+  model: "gpt-4o"                       # Multimodal model name
+
+# HTTP/SSE Server bind configurations.
+# Only used when serving via HTTP transport.
+server:
+  host: "127.0.0.1"
+  port: 8000

document_rag_mcp-0.1.0/docs/architecture.md

@@ -0,0 +1,38 @@
+# System Architecture
+
+The Document RAG MCP server is designed as a modular, lightweight, and performant pipeline.
+
+## System Diagram
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                         MCP Server (FastMCP)                    │
+│  Transport: stdio | streamable-http                             │
+│                                                                 │
+│  Tools:     search, list_collections, get_document_content,     │
+│             get_document_original, ingest_now                   │
+└──────────┬──────────────────────────────┬───────────────────────┘
+           │                              │
+     ┌─────▼──────┐              ┌────────▼────────┐
+     │  Ingestion  │              │   Search        │
+     │  Pipeline   │              │   Engine        │
+     │             │              │                 │
+     │  extract →  │              │  embed query →  │
+     │  chunk →    │              │  vector query → │
+     │  hash →     │              │  merge & rank   │
+     │  upsert     │              └────────┬────────┘
+     └─────┬──────┘                        │
+           │                               │
+     ┌─────▼──────┐                  ┌─────▼──────┐
+     │State Store │                  │Vector Store│
+     │  (SQLite)  │                  │ (ChromaDB) │
+     └────────────┘                  └────────────┘
+```
+
+## Modular Description
+
+- **Extraction (`extractor.py`)**: Responsible for reading plain text, parsing frontmatter/headings in markdown, and extraction from PDFs via PyMuPDF. It uses font sizes and tags in the PDF layout to detect headers.
+- **Chunking (`chunker.py`)**: Uses `chonkie` to partition text into tokens. Markdown is recursively split at section headers and paragraph shifts, while TXT/PDF is semantically chunked.
+- **Incremental Pipeline (`pipeline.py`)**: Checks file-level hashes (SHA-256) and stores them in SQLite. If a file is modified, it computes new chunk hashes, maps existing chunk vectors from ChromaDB, and only requests embeddings for new/modified chunks, saving API tokens.
+- **Search Engine (`engine.py`)**: Performs semantic searches by generating a vector representation of the query and querying ChromaDB. For multi-collection queries, it merges and ranks results using ascending order of L2 distance.
+- **Security Boundaries**: Path operations inside the MCP tools (`get_document_content`, `get_document_original`) validate that the target files reside within one of the collection directories before performing file system reads, protecting the server host from arbitrary path traversal attacks.

document_rag_mcp-0.1.0/docs/cli.md

@@ -0,0 +1,53 @@
+# CLI Reference
+
+The `document-rag-mcp` CLI exposes subcommands to run the RAG server, test searches, and trigger manual index scanning.
+
+## Global Options
+
+- `--config`, `-c`: Path to the YAML configuration file. Can also be set via the `DOCRAG_CONFIG` environment variable.
+- `--chunking-model`: Override the local model name for semantic boundary splits. Can also be set via the `DOCRAG_CHUNKING_MODEL` environment variable.
+- `--help`: Show the help message and exit.
+
+## Subcommands
+
+### `serve`
+Starts the Model Context Protocol (MCP) server.
+
+- `--transport`: Choose `stdio` (default) or `http` (SSE).
+- `--host`: Bind host for SSE transport (default `127.0.0.1`).
+- `--port`: Bind port for SSE transport (default `8000`).
+
+**Example:**
+```bash
+document-rag-mcp serve --transport stdio
+```
+
+### `ingest`
+Trigger a one-shot synchronous recursive scan and index of files in all collections (or a specific collection). This command prunes entries for deleted files.
+
+- `--collection`, `-c`: Limit the scan to a specific collection by name.
+
+**Example:**
+```bash
+document-rag-mcp ingest --collection "project-docs"
+```
+
+### `search`
+Executes a semantic search against the indexed collections and prints formatted results to the terminal.
+
+- `QUERY` (Argument, Required): The semantic search query text.
+- `--collection`, `-c`: Filter search results to a specific collection by name.
+- `--top-k`, `-k`: The number of nearest matches to display (default 5).
+
+**Example:**
+```bash
+document-rag-mcp search "how to configure the pipeline" -k 3
+```
+
+### `collections`
+Lists all collections specified in the configuration file, showing their configured directories, search file glob patterns, and the total count of indexed chunks.
+
+**Example:**
+```bash
+document-rag-mcp collections
+```

document_rag_mcp-0.1.0/docs/configuration.md

@@ -0,0 +1,40 @@
+# Configuration Reference
+
+The server loads configuration from a YAML file. You can specify the config file path using the `-c`/`--config` CLI flag or the `DOCRAG_CONFIG` environment variable.
+
+## Environment Variables Override
+
+All configuration fields can be overridden using environment variables prefixed with `DOCRAG_` and using a double underscore `__` for nesting.
+
+**Examples:**
+- Override storage directory: `DOCRAG_STORAGE__DATA_DIR="/tmp/data"`
+- Override embedding model: `DOCRAG_EMBEDDING__MODEL="text-embedding-3-small"`
+- Override vision enabled: `DOCRAG_VISION__ENABLED="true"`
+
+## Detailed Configuration Options
+
+### Collections Settings
+Configure folders and matching patterns:
+- `name`: Unique name of the collection (conform to alphanumeric rules).
+- `paths`: Absolute paths to folders/files to index.
+- `file_patterns`: Glob patterns, e.g. `["*.txt", "*.md", "*.pdf"]`.
+
+### Embedding Settings
+Configure the OpenAI-compatible embedding API:
+- `base_url`: Target endpoint (e.g. `http://localhost:8080/v1` for lemonade).
+- `api_key`: Authorization API Key (use "unused" if endpoint does not require one).
+- `model`: Embedding model name.
+- `dimensions`: Vector dimensions size (e.g., 768 for Gemma).
+- `batch_size`: Maximum texts sent in a single batch request.
+
+### Local Chunking Settings
+Configure document splitting limits:
+- `max_chunk_size`: Maximum number of tokens per chunk.
+- `similarity_threshold`: Threshold for semantic boundary splits.
+- `local_model`: Local model name (e.g. `all-MiniLM-L6-v2`) used for chunking boundaries.
+
+### Vision Settings (Optional)
+Configure scanned PDF page-to-image extraction:
+- `enabled`: Set `true` to enable multimodal OCR fallback.
+- `base_url`: OpenAI-compatible vision completion endpoint.
+- `model`: Multimodal model name (e.g. `gpt-4o`).

document_rag_mcp-0.1.0/docs/getting-started.md

@@ -0,0 +1,58 @@
+# Getting Started
+
+Follow these steps to install, configure, and run the Document RAG MCP server.
+
+## Installation
+
+This project uses `uv` for python dependency management. Ensure you have `uv` installed, then synchronize the environment:
+
+```bash
+git clone https://github.com/user/document-rag-mcp.git
+cd document-rag-mcp
+uv sync --group dev
+```
+
+## First Configuration
+
+Copy the example configuration file and adjust it to your directories:
+
+```bash
+cp config.example.yaml config.yaml
+```
+
+Edit `config.yaml` to specify the folders you want to watch.
+
+## Quick CLI Usage
+
+You can test the ingestion and run semantic searches directly from the command line:
+
+### 1. Ingest Documents
+Run a one-shot ingestion scan to populate the vector store:
+```bash
+uv run document-rag-mcp ingest
+```
+
+### 2. Run a Test Search
+Query the indexed collections:
+```bash
+uv run document-rag-mcp search "What is project antigravity?"
+```
+
+### 3. Show Collection Statistics
+Check chunk counts and file listings:
+```bash
+uv run document-rag-mcp collections
+```
+
+## Running the MCP Server
+
+Start the server using `stdio` transport (default) for integration with LLM clients:
+
+```bash
+uv run document-rag-mcp serve --transport stdio
+```
+
+For HTTP/SSE integration, specify the transport and bindings:
+```bash
+uv run document-rag-mcp serve --transport http --host 127.0.0.1 --port 8000
+```

document_rag_mcp-0.1.0/docs/index.md

@@ -0,0 +1,14 @@
+# Document RAG MCP Server
+
+Welcome to the **document-rag-mcp** server documentation!
+
+This Model Context Protocol (MCP) server enables semantic search and content extraction over text, Markdown, and PDF documents. It works fully in-process without requiring external database servers, using ChromaDB as the vector store and SQLite as the ingestion state tracker.
+
+## Key Features
+
+- **Recursive Folder Scanning**: Configured folders are recursively scanned on startup and then monitored in real-time via inotify (`watchfiles`).
+- **Incremental Indexing**: Uses content hashing (SHA-256) at both the file and chunk levels. Files that have not changed are skipped completely on startup, and modified files only re-embed chunks that actually changed.
+- **Auto-Pruning**: Automatically detects when files are deleted from the disk (both while running and when offline) and prunes them from the index.
+- **Multimodal PDF Processing**: Detects scanned or text-less PDF pages and routes them through an optional vision-capable LLM to extract text.
+- **MCP Native**: Exposes tools for semantic search, collection stats, metadata analysis, and full document text/binary content retrieval.
+- **Secure Boundaries**: Strictly validates all path parameters against configured collections folders to protect against directory traversal attacks.

document_rag_mcp-0.1.0/mkdocs.yml

@@ -0,0 +1,35 @@
+site_name: document-rag-mcp Documentation
+theme:
+  name: material
+  palette:
+    - scheme: slate
+      primary: teal
+      accent: teal
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+    - scheme: default
+      primary: teal
+      accent: teal
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+  features:
+    - navigation.sections
+    - navigation.expand
+    - content.code.copy
+
+plugins:
+  - search
+  - mkdocstrings:
+      default_handler: python
+      handlers:
+        python:
+          paths: [src]
+
+nav:
+  - Home: index.md
+  - Getting Started: getting-started.md
+  - Configuration: configuration.md
+  - Architecture: architecture.md
+  - CLI Reference: cli.md

document_rag_mcp-0.1.0/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "document-rag-mcp"
+version = "0.1.0"
+description = "RAG MCP Server — watches folders, chunks documents, serves semantic search via MCP"
+readme = "README.md"
+license = { text = "AGPL-3.0-or-later" }
+requires-python = ">=3.11"
+dependencies = [
+    "mcp[cli]>=1.27,<2",
+    "chromadb>=1.5",
+    "openai>=1.80",
+    "chonkie[semantic]>=1.0",
+    "pymupdf>=1.25",
+    "watchfiles>=1.0",
+    "pydantic>=2.0",
+    "pydantic-settings>=2.0",
+    "click>=8.0",
+    "pyyaml>=6.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "pytest-cov>=6.0",
+    "ruff>=0.9",
+]
+docs = [
+    "mkdocs-material>=9.6",
+    "mkdocstrings[python]>=0.27",
+]
+
+[project.scripts]
+document-rag-mcp = "document_rag_mcp.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/document_rag_mcp"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+src = ["src"]

document_rag_mcp-0.1.0/src/document_rag_mcp/__init__.py

@@ -0,0 +1,2 @@
+# document-rag-mcp package
+__version__ = "0.1.0"