chunksilo 2.0.0__py3-none-any.whl

chunksilo/server.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""
+MCP server for querying documentation using RAG.
+Returns raw document chunks for the calling LLM to synthesize.
+"""
+import argparse
+import asyncio
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Annotated, Any
+
+from pydantic import Field
+from mcp.server.fastmcp import FastMCP
+
+# Log file configuration
+LOG_FILE = "mcp.log"
+LOG_MAX_SIZE_MB = 10
+LOG_MAX_SIZE_BYTES = LOG_MAX_SIZE_MB * 1024 * 1024
+
+# Module-level state (set during initialization)
+_server_config_path: Path | None = None
+_mcp: FastMCP | None = None
+
+
+def _rotate_log_if_needed():
+    """Rotate log file if it exists and is over the size limit."""
+    log_path = Path(LOG_FILE)
+    if log_path.exists() and log_path.stat().st_size > LOG_MAX_SIZE_BYTES:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+        process_id = os.getpid()
+        rotated_name = f"mcp_{timestamp}_{process_id}.log"
+        rotated_path = log_path.parent / rotated_name
+        log_path.rename(rotated_path)
+        log_path.touch()
+
+
+def _setup_logging():
+    """Configure logging for MCP server mode - file only, no stdout/stderr."""
+    _rotate_log_if_needed()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        force=True,
+        handlers=[
+            logging.FileHandler(LOG_FILE, encoding="utf-8"),
+        ],
+    )
+    logging.getLogger("llama_index.readers.confluence").setLevel(logging.WARNING)
+
+
+def _create_server() -> FastMCP:
+    """Create and configure the MCP server with tools."""
+    from .search import run_search
+
+    mcp = FastMCP("llamaindex-docs-rag")
+
+    @mcp.tool()
+    async def search_docs(
+        query: Annotated[str, Field(description="Search query text")],
+        date_from: Annotated[str | None, Field(description="Optional start date filter (YYYY-MM-DD format, inclusive)")] = None,
+        date_to: Annotated[str | None, Field(description="Optional end date filter (YYYY-MM-DD format, inclusive)")] = None,
+    ) -> dict[str, Any]:
+        """Search across all your indexed documentation using a natural language query."""
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(
+            None, lambda: run_search(query, date_from, date_to, config_path=_server_config_path)
+        )
+
+    return mcp
+
+
+def run_server(config_path: Path | None = None):
+    """Start the MCP server."""
+    global _server_config_path, _mcp
+
+    if config_path:
+        _server_config_path = config_path
+        os.environ["CHUNKSILO_CONFIG"] = str(config_path)
+
+    _mcp = _create_server()
+    _mcp.run()
+
+
+def main():
+    """Entry point for the chunksilo-mcp command."""
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    parser = argparse.ArgumentParser(
+        prog="chunksilo-mcp",
+        description="Run ChunkSilo MCP server (stdio transport)",
+    )
+    parser.add_argument("--config", help="Path to config.yaml")
+    args = parser.parse_args()
+
+    # Configure logging BEFORE importing anything that uses logging
+    _setup_logging()
+
+    logger = logging.getLogger(__name__)
+    logger.info("Starting ChunkSilo MCP server")
+
+    config_path = Path(args.config) if args.config else None
+    run_server(config_path)
+
+
+if __name__ == "__main__":
+    main()

chunksilo-2.0.0.dist-info/METADATA

@@ -0,0 +1,366 @@
+Metadata-Version: 2.4
+Name: chunksilo
+Version: 2.0.0
+Summary: Local RAG-based semantic document search with MCP server interface
+Author: Fredrik Reveny
+License-Expression: Apache-2.0
+Project-URL: Repository, https://github.com/Chetic/chunksilo
+Project-URL: Issues, https://github.com/Chetic/chunksilo/issues
+Keywords: rag,mcp,semantic-search,document-search,llm
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: llama-index<1,>=0.10.0
+Requires-Dist: llama-index-readers-file<1,>=0.1.0
+Requires-Dist: llama-index-embeddings-fastembed<1,>=0.5.0
+Requires-Dist: llama-index-retrievers-bm25<1,>=0.1.0
+Requires-Dist: pillow<11,>=10.3.0
+Requires-Dist: pypdf<7,>=5.1.0
+Requires-Dist: python-docx<2,>=1.1.0
+Requires-Dist: mcp<2,>=1.0.0
+Requires-Dist: python-dotenv<2,>=1.0.0
+Requires-Dist: huggingface-hub<2,>=0.22.0
+Requires-Dist: flashrank<1,>=0.1.0
+Requires-Dist: fastembed<1,>=0.5.0
+Requires-Dist: pyyaml<7,>=6.0
+Provides-Extra: confluence
+Requires-Dist: llama-index-readers-confluence<1,>=0.6.0; extra == "confluence"
+Provides-Extra: test
+Requires-Dist: pytest<9,>=7.4.0; extra == "test"
+Requires-Dist: requests<3,>=2.31.0; extra == "test"
+Dynamic: license-file
+
+# ChunkSilo MCP Server
+
+ChunkSilo is like a local Google for your documents. It uses semantic search — matching by meaning rather than exact keywords — so your LLM can find relevant information across all your files even when the wording differs from your query. Point it at your PDFs, Word docs, Markdown, and text files, and it builds a fully searchable index locally on your machine.
+
+## Overview
+
+- **No permissions headache**: Each user indexes only the files they already have access to. No centralized access-control system to build or maintain — document permissions stay exactly where they are.
+- **No infrastructure required**: Runs entirely on the user's own machine as an MCP server. Nothing to deploy, no servers to manage.
+- **Easy to set up**: Any user with an MCP-compatible LLM client can install, point at their document directories, and have everything indexed and searchable.
+- **Works with what you have**: Supports PDF, DOCX, DOC, Markdown, and TXT from local folders, network drives, or shared mounts.
+
+## Features
+
+- **Local indexing and search**: All indexing and search runs on your machine with bundled models — ChunkSilo itself makes no external network calls when `offline: true`. Note: search results are passed to your MCP client's LLM, which may be cloud-hosted.
+- **Incremental indexing**: Only reindexes new or changed files, so re-runs are fast even on large document collections.
+- **Heading-aware navigation**: Extracts headings from PDFs, Word docs, and Markdown so results include the full heading path (e.g. "Chapter 3 > Setup > Prerequisites").
+- **Date filtering and recency boost**: Search within a date range or let recent documents rank higher automatically.
+- **Dual retrieval**: Returns both meaning-based chunk matches and keyword-based filename matches separately, so file lookups don't get buried by unrelated content.
+- **Multi-directory with per-folder rules**: Index multiple directories with individual include/exclude glob patterns — useful for shared drives with mixed content.
+- **Confluence integration**: Optionally searches your Confluence instance alongside local files, with results returned in the same format.
+- **Source links**: Each result includes a clickable link back to the source file or Confluence page in supported MCP clients.
+
+## Installation
+
+### Option A: Install from PyPI (Recommended)
+
+Requires Python 3.11 or later. Models are downloaded automatically on first run (~250MB). The first run may appear to pause while models download — this is normal.
+
+```bash
+pip install chunksilo
+
+# Or with Confluence support:
+pip install chunksilo[confluence]
+```
+
+Then:
+1. **Create** a config file at `~/.config/chunksilo/config.yaml` (see [Configuration](#configuration))
+2. **Build** the index: `chunksilo --build-index`
+3. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration))
+
+### Option B: Offline Bundle
+
+A self-contained package with pre-downloaded models, ideal for air-gapped environments or systems without Python installed.
+
+Download from the [Releases page](https://github.com/Chetic/chunksilo/releases):
+
+1. **Download** the `chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz` file
+2. **Extract** and install:
+
+```bash
+tar -xzf chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz
+cd chunksilo
+./setup.sh
+```
+
+3. **Edit** `config.yaml` to set your document directories
+4. **Build** the index: `./venv/bin/chunksilo --build-index`
+5. **Configure** your MCP client (see [MCP Client Configuration](#mcp-client-configuration))
+
+## Configuration
+
+ChunkSilo uses a single configuration file: `config.yaml`
+
+### Configuration File
+
+Edit `config.yaml` to configure your settings:
+
+```yaml
+# Indexing settings - used by chunksilo --build-index
+indexing:
+  directories:
+    - "./data"
+    - "/mnt/nfs/shared-docs"
+    - path: "/mnt/samba/engineering"
+      include: ["**/*.pdf", "**/*.md"]
+      exclude: ["**/archive/**"]
+  chunk_size: 1600
+  chunk_overlap: 200
+
+# Retrieval settings - used when searching
+retrieval:
+  embed_top_k: 20
+  rerank_top_k: 5
+  score_threshold: 0.1
+
+# Confluence integration (optional)
+confluence:
+  url: "https://confluence.example.com"
+  username: "your-username"
+  api_token: "your-api-token"
+
+# Storage paths (usually don't need to change)
+storage:
+  storage_dir: "./storage"
+  model_cache_dir: "./models"
+```
+
+All settings are optional and have sensible defaults.
+
+### Configuration Reference
+
+#### Indexing Settings
+
+| Setting | Default | Description |
+| :--- | :--- | :--- |
+| `indexing.directories` | `["./data"]` | List of directories to index (strings or objects) |
+| `indexing.chunk_size` | `1600` | Maximum size of text chunks |
+| `indexing.chunk_overlap` | `200` | Overlap between adjacent chunks |
+
+**Per-directory options** (when using object format):
+
+| Option | Default | Description |
+| :--- | :--- | :--- |
+| `path` | (required) | Directory path to index |
+| `include` | `["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"]` | Glob patterns for files to include |
+| `exclude` | `[]` | Glob patterns for files to exclude |
+| `recursive` | `true` | Whether to recurse into subdirectories |
+| `enabled` | `true` | Whether to index this directory |
+
+#### Retrieval Settings
+
+| Setting | Default | Description |
+| :--- | :--- | :--- |
+| `retrieval.embed_model_name` | `BAAI/bge-small-en-v1.5` | Embedding model for vector search |
+| `retrieval.embed_top_k` | `20` | Candidates from vector search before reranking |
+| `retrieval.rerank_model_name` | `ms-marco-MiniLM-L-12-v2` | Reranker model |
+| `retrieval.rerank_top_k` | `5` | Final results after reranking |
+| `retrieval.rerank_candidates` | `100` | Maximum candidates sent to reranker |
+| `retrieval.score_threshold` | `0.1` | Minimum score (0.0-1.0) for results |
+| `retrieval.recency_boost` | `0.3` | Recency boost weight (0.0-1.0) |
+| `retrieval.recency_half_life_days` | `365` | Days until recency boost halves |
+| `retrieval.bm25_similarity_top_k` | `10` | Files returned by BM25 filename search |
+| `retrieval.offline` | `false` | Prevent ML library network requests |
+
+#### Confluence Settings (optional)
+
+> **Note:** Confluence integration requires the optional dependency. Install with: `pip install chunksilo[confluence]`
+
+| Setting | Default | Description |
+| :--- | :--- | :--- |
+| `confluence.url` | `""` | Confluence base URL (empty = disabled) |
+| `confluence.username` | `""` | Confluence username |
+| `confluence.api_token` | `""` | Confluence API token |
+| `confluence.timeout` | `10.0` | Request timeout in seconds |
+| `confluence.max_results` | `30` | Maximum results per search |
+
+#### SSL Settings (optional)
+
+| Setting | Default | Description |
+| :--- | :--- | :--- |
+| `ssl.ca_bundle_path` | `""` | Path to custom CA bundle file |
+
+#### Storage Settings
+
+| Setting | Default | Description |
+| :--- | :--- | :--- |
+| `storage.storage_dir` | `./storage` | Directory for vector index and state |
+| `storage.model_cache_dir` | `./models` | Directory for model cache |
+
+## CLI Usage
+
+The `chunksilo` command provides indexing, searching, and model management:
+
+```bash
+# Build or update the search index
+chunksilo --build-index
+
+# Search for documents
+chunksilo "your search query"
+
+# Search with date filtering
+chunksilo "quarterly report" --date-from 2024-01-01 --date-to 2024-03-31
+
+# Output results as JSON
+chunksilo "search query" --json
+
+# Show verbose output (model loading, search stats)
+chunksilo "search query" --verbose
+
+# Pre-download ML models (useful before going offline)
+chunksilo --download-models
+
+# Use a custom config file
+chunksilo --build-index --config /path/to/config.yaml
+```
+
+### CLI Options
+
+| Option | Description |
+| :--- | :--- |
+| `query` | Search query text (positional argument) |
+| `--build-index` | Build or update the search index, then exit |
+| `--download-models` | Download required ML models, then exit |
+| `--date-from` | Start date filter (YYYY-MM-DD format, inclusive) |
+| `--date-to` | End date filter (YYYY-MM-DD format, inclusive) |
+| `--json` | Output results as JSON instead of formatted text |
+| `-v, --verbose` | Show diagnostic messages (model loading, search stats) |
+| `--config` | Path to config.yaml (overrides auto-discovery) |
+
+## MCP Client Configuration
+
+Configure your MCP client to run ChunkSilo. Below are examples for common clients.
+
+> **Note:** For PyPI installs, use `chunksilo-mcp` directly. For offline bundles, use the full path `/path/to/chunksilo/venv/bin/chunksilo-mcp`. You can find the PyPI-installed binary location with `which chunksilo-mcp`.
+
+### Claude Code
+
+Add chunksilo as an MCP server using the CLI:
+
+**PyPI install:**
+```bash
+claude mcp add chunksilo --scope user -- chunksilo-mcp --config ~/.config/chunksilo/config.yaml
+```
+
+**Offline bundle:**
+```bash
+claude mcp add chunksilo --scope user -- /path/to/chunksilo/venv/bin/chunksilo-mcp --config /path/to/chunksilo/config.yaml
+```
+
+Verify it's connected:
+
+```bash
+claude mcp list
+```
+
+### Claude Desktop
+
+Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows):
+
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"]
+    }
+  }
+}
+```
+
+**Offline bundle:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"]
+    }
+  }
+}
+```
+
+### Cline (VS Code Extension)
+
+Add to `cline_mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/`):
+
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"],
+      "disabled": false,
+      "autoApprove": []
+    }
+  }
+}
+```
+
+**Offline bundle:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"],
+      "disabled": false,
+      "autoApprove": []
+    }
+  }
+}
+```
+
+### Roo Code (VS Code Extension)
+
+Add to `mcp_settings.json` (typically in `~/.config/Code/User/globalStorage/rooveterinaryinc.roo-cline/settings/`):
+
+**PyPI install:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "chunksilo-mcp",
+      "args": ["--config", "/path/to/config.yaml"]
+    }
+  }
+}
+```
+
+**Offline bundle:**
+```json
+{
+  "mcpServers": {
+    "chunksilo": {
+      "command": "/path/to/chunksilo/venv/bin/chunksilo-mcp",
+      "args": ["--config", "/path/to/chunksilo/config.yaml"]
+    }
+  }
+}
+```
+
+## Troubleshooting
+
+- **Index missing**: Run `chunksilo --build-index` (PyPI install) or `./venv/bin/chunksilo --build-index` (offline bundle).
+- **Retrieval errors**: Check paths in your MCP client configuration.
+- **Offline mode**: PyPI installs default to `offline: false` (models auto-download). The offline bundle includes pre-downloaded models and sets `offline: true`. Set `retrieval.offline: true` in `config.yaml` to prevent network calls after initial model download.
+- **Confluence Integration**: Install with `pip install chunksilo[confluence]`, then set `confluence.url`, `confluence.username`, and `confluence.api_token` in `config.yaml`.
+- **Custom CA Bundle**: Set `ssl.ca_bundle_path` in `config.yaml` for custom certificates.
+- **Network mounts**: Unavailable directories are skipped with a warning; indexing continues with available directories.
+- **Legacy .doc files**: Requires LibreOffice to be installed for automatic conversion to .docx. If LibreOffice is not found, .doc files are skipped with a warning. Full heading extraction is supported.
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE) for details.

chunksilo-2.0.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+chunksilo/__init__.py,sha256=Ph1w-3A5CKoGdxzUNVlrB483eOM5JxAQN8K88-6HbuY,121
+chunksilo/__main__.py,sha256=eY8-KfJfBz0nibDPY_jv2RvkLXEx7ZDSPRWiDJb7PpY,39
+chunksilo/cfgload.py,sha256=A7ab2RkeYQhFTTvn56I900lWpXCqXb8ao8DGYeUai1U,4359
+chunksilo/cli.py,sha256=ZKXzW-HOmnLXC4Ynu-D86XIbp7bdQ_OYVHuHFVlj41U,4170
+chunksilo/confluence_html_formatter.py,sha256=D8pb5TCrai6exIqeajH49y4D_t0jfQkmI6aNm4BzPIg,2828
+chunksilo/index.py,sha256=ECSTH8c0ZhnTxAHxM7dcCfFOGTk99STHuyDhc3uMCO0,51618
+chunksilo/search.py,sha256=8_6zuvyVdZHNMCBHrK3NcP0Ct7oYeCSLuoOHT8RxD0o,28722
+chunksilo/server.py,sha256=PhQWMvEGb3UqWwk0tm44kTOnKH2NH4NmshnKx11xwh8,3374
+chunksilo-2.0.0.dist-info/licenses/LICENSE,sha256=kda7NTahQy3nKvLe-LGIVCdI-qePTyqNNx2e6HLVH3k,10766
+chunksilo-2.0.0.dist-info/licenses/NOTICE,sha256=58GCfasR-XT7RPvz9OoWb4NrPg-3AqEwzw9FJOhngZo,1228
+chunksilo-2.0.0.dist-info/METADATA,sha256=AA4R48_d6h7V89dHM9hb0av53WlfrDKZzALfOZ0jywU,13360
+chunksilo-2.0.0.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+chunksilo-2.0.0.dist-info/entry_points.txt,sha256=TfgrfBQyIRHMG5NFaXR-owDLsjHBCxjJ4YGIGBnVzbQ,87
+chunksilo-2.0.0.dist-info/top_level.txt,sha256=xfAH0GhTfZbwkbi_DgUA5keNP-osqVNcoZjwkhpwVms,10
+chunksilo-2.0.0.dist-info/RECORD,,

chunksilo-2.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

chunksilo-2.0.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+chunksilo = chunksilo.cli:main
+chunksilo-mcp = chunksilo.server:main