chunksilo 2.3.4__tar.gz → 3.0.0__tar.gz

chunksilo-3.0.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: chunksilo
+Version: 3.0.0
+Summary: Local RAG-based semantic document search with MCP server interface
+Author: Fredrik Reveny
+License-Expression: Apache-2.0
+Project-URL: Repository, https://github.com/Chetic/chunksilo
+Project-URL: Issues, https://github.com/Chetic/chunksilo/issues
+Keywords: rag,mcp,semantic-search,document-search,llm
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: llama-index<1,>=0.10.0
+Requires-Dist: llama-index-readers-file<1,>=0.1.0
+Requires-Dist: llama-index-embeddings-fastembed<1,>=0.5.0
+Requires-Dist: llama-index-retrievers-bm25<1,>=0.1.0
+Requires-Dist: pillow<11,>=10.3.0
+Requires-Dist: pypdf<7,>=5.1.0
+Requires-Dist: python-docx<2,>=1.1.0
+Requires-Dist: mcp<2,>=1.10.0
+Requires-Dist: huggingface-hub<2,>=0.22.0
+Requires-Dist: flashrank<1,>=0.1.0
+Requires-Dist: fastembed<1,>=0.5.0
+Requires-Dist: pyyaml<7,>=6.0
+Requires-Dist: llama-index-readers-confluence<1,>=0.6.0
+Requires-Dist: jira<4,>=3.5.0
+Provides-Extra: confluence
+Provides-Extra: jira
+Provides-Extra: test
+Requires-Dist: pytest<9,>=7.4.0; extra == "test"
+Requires-Dist: pytest-cov>=4.0; extra == "test"
+Requires-Dist: requests<3,>=2.31.0; extra == "test"
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/Chetic/chunksilo/main/chunksilo.png" alt="ChunkSilo Logo" width="500">
+</p>
+
+<p align="center">
+  <img src="demo/demo.gif" alt="ChunkSilo terminal demo" width="720">
+</p>
+
+# ChunkSilo MCP Server
+
+ChunkSilo is like a local Google for your documents. It uses semantic search — matching by meaning rather than exact keywords — so your LLM can find relevant information across all your files even when the wording differs from your query. Point it at your PDFs, Word docs, Markdown, and text files, and it builds a fully searchable index locally on your machine.
+
+- Runs entirely on your machine — no servers, no infrastructure
+- Semantic search + keyword filename matching across PDF, DOCX, DOC, Markdown, and TXT
+- Incremental indexing — only reprocesses new or changed files, and never writes inside an indexed directory
+- Revision-aware indexing — files that are revisions of one document (`Spec_v2.pdf`, `.../Rev B/Spec.docx`) are grouped and only the newest is indexed; review-comment copies are skipped
+- Heading-aware results with source links back to the original file, spread across documents instead of one document's every chunk
+- Date filtering and recency boosting
+- Files indexed from a mounted network share can be presented at the share's own location (an `smb://` URI plus the Windows UNC path)
+- Serves MCP over stdio to a local client, or over streamable-http behind a reverse proxy of your own (oauth2-proxy + KeyCloak walkthrough included)
+- Optional Confluence and Jira integrations (supports Cloud and Server/Data Center)
+
+## Installation
+
+### Option A: Install from PyPI (Recommended)
+
+Requires Python 3.11 or later. Models are downloaded automatically on first run (~250MB). The first run may appear to pause while models download — this is normal.
+
+```bash
+pip install chunksilo
+```
+
+Confluence and Jira support is included by default — just provide a config file
+to enable them. (`pip install chunksilo[confluence,jira]` still works as an alias
+for backward compatibility.)
+
+Then:
+1. **Create** a config file at `~/.config/chunksilo/config.yaml` (see [Configuration](#configuration))
+2. **Build** the index: `chunksilo --build-index`
+3. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md))
+
+### Option B: Offline Bundle
+
+A self-contained package with pre-downloaded models, ideal for air-gapped environments or systems without Python installed.
+
+Download from the [Releases page](https://github.com/Chetic/chunksilo/releases):
+
+1. **Download** the `chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz` file
+2. **Extract** and install:
+
+```bash
+tar -xzf chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz
+cd chunksilo
+./setup.sh
+```
+
+3. **Edit** `config.yaml` to set your document directories
+4. **Build** the index: `./venv/bin/chunksilo --build-index`
+5. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md))
+
+## Configuration
+
+ChunkSilo reads one file, `config.yaml`, taken from `--config`, else the
+`CHUNKSILO_CONFIG` environment variable, else `./config.yaml`, else
+`~/.config/chunksilo/config.yaml`. Every setting is optional and
+`chunksilo --dump-defaults` prints them all with their defaults.
+
+```yaml
+indexing:
+  directories:
+    - "./data"
+    - "/mnt/docs"
+
+retrieval:
+  rerank_top_k: 5
+
+storage:
+  storage_dir: "./storage"
+  model_cache_dir: "./models"
+```
+
+The full reference — per-directory filters, revision handling, retrieval
+tuning, Confluence and Jira, share locations, the HTTP transport, and what
+changed for users upgrading from 2.x — is in
+[docs/configuration.md](docs/configuration.md).
+
+## The `search_docs` tool
+
+`search_docs` answers with the files whose names matched and the individual
+chunks that carry the answer, each located by a URI, the page or line it came
+from, and its heading path:
+
+```json
+{
+  "matched_files": [
+    { "uri": "file:///docs/database-configuration.docx", "unc": null, "score": 0.8432 }
+  ],
+  "num_matched_files": 1,
+  "chunks": [
+    {
+      "text": "To configure the database connection, set the DATABASE_URL environment variable...",
+      "score": 0.912,
+      "location": {
+        "uri": "file:///docs/setup-guide.pdf",
+        "unc": null,
+        "page": 12,
+        "line": null,
+        "heading_path": ["Getting Started", "Configuration", "Database"]
+      }
+    }
+  ],
+  "num_chunks": 1,
+  "query": "how to configure the database",
+  "retrieval_time": "0.42s"
+}
+```
+
+See [docs/tools.md](docs/tools.md) for every field, including how files on a
+mounted network share are presented.
+
+## Documentation
+
+- [docs/configuration.md](docs/configuration.md) — the config file, where it is discovered, and a reference for every setting.
+- [docs/cli.md](docs/cli.md) — `chunksilo` and `chunksilo-mcp` options, including `--check-files` for explaining why a file is or is not indexed.
+- [docs/tools.md](docs/tools.md) — the `search_docs` result shape.
+- [docs/mcp-clients.md](docs/mcp-clients.md) — client setup for Claude Code, Claude Desktop, Cline, Roo Code and opencode, over stdio or HTTP.
+- [docs/reverse-proxy.md](docs/reverse-proxy.md) — sharing an instance behind an authenticating reverse proxy, with oauth2-proxy and KeyCloak group-based access.
+- [docs/troubleshooting.md](docs/troubleshooting.md) — indexing surprises, network mounts, offline mode and more.
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

chunksilo-3.0.0/README.md

@@ -0,0 +1,132 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/Chetic/chunksilo/main/chunksilo.png" alt="ChunkSilo Logo" width="500">
+</p>
+
+<p align="center">
+  <img src="demo/demo.gif" alt="ChunkSilo terminal demo" width="720">
+</p>
+
+# ChunkSilo MCP Server
+
+ChunkSilo is like a local Google for your documents. It uses semantic search — matching by meaning rather than exact keywords — so your LLM can find relevant information across all your files even when the wording differs from your query. Point it at your PDFs, Word docs, Markdown, and text files, and it builds a fully searchable index locally on your machine.
+
+- Runs entirely on your machine — no servers, no infrastructure
+- Semantic search + keyword filename matching across PDF, DOCX, DOC, Markdown, and TXT
+- Incremental indexing — only reprocesses new or changed files, and never writes inside an indexed directory
+- Revision-aware indexing — files that are revisions of one document (`Spec_v2.pdf`, `.../Rev B/Spec.docx`) are grouped and only the newest is indexed; review-comment copies are skipped
+- Heading-aware results with source links back to the original file, spread across documents instead of one document's every chunk
+- Date filtering and recency boosting
+- Files indexed from a mounted network share can be presented at the share's own location (an `smb://` URI plus the Windows UNC path)
+- Serves MCP over stdio to a local client, or over streamable-http behind a reverse proxy of your own (oauth2-proxy + KeyCloak walkthrough included)
+- Optional Confluence and Jira integrations (supports Cloud and Server/Data Center)
+
+## Installation
+
+### Option A: Install from PyPI (Recommended)
+
+Requires Python 3.11 or later. Models are downloaded automatically on first run (~250MB). The first run may appear to pause while models download — this is normal.
+
+```bash
+pip install chunksilo
+```
+
+Confluence and Jira support is included by default — just provide a config file
+to enable them. (`pip install chunksilo[confluence,jira]` still works as an alias
+for backward compatibility.)
+
+Then:
+1. **Create** a config file at `~/.config/chunksilo/config.yaml` (see [Configuration](#configuration))
+2. **Build** the index: `chunksilo --build-index`
+3. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md))
+
+### Option B: Offline Bundle
+
+A self-contained package with pre-downloaded models, ideal for air-gapped environments or systems without Python installed.
+
+Download from the [Releases page](https://github.com/Chetic/chunksilo/releases):
+
+1. **Download** the `chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz` file
+2. **Extract** and install:
+
+```bash
+tar -xzf chunksilo-vX.Y.Z-manylinux_2_34_x86_64.tar.gz
+cd chunksilo
+./setup.sh
+```
+
+3. **Edit** `config.yaml` to set your document directories
+4. **Build** the index: `./venv/bin/chunksilo --build-index`
+5. **Configure** your MCP client (see [docs/mcp-clients.md](docs/mcp-clients.md))
+
+## Configuration
+
+ChunkSilo reads one file, `config.yaml`, taken from `--config`, else the
+`CHUNKSILO_CONFIG` environment variable, else `./config.yaml`, else
+`~/.config/chunksilo/config.yaml`. Every setting is optional and
+`chunksilo --dump-defaults` prints them all with their defaults.
+
+```yaml
+indexing:
+  directories:
+    - "./data"
+    - "/mnt/docs"
+
+retrieval:
+  rerank_top_k: 5
+
+storage:
+  storage_dir: "./storage"
+  model_cache_dir: "./models"
+```
+
+The full reference — per-directory filters, revision handling, retrieval
+tuning, Confluence and Jira, share locations, the HTTP transport, and what
+changed for users upgrading from 2.x — is in
+[docs/configuration.md](docs/configuration.md).
+
+## The `search_docs` tool
+
+`search_docs` answers with the files whose names matched and the individual
+chunks that carry the answer, each located by a URI, the page or line it came
+from, and its heading path:
+
+```json
+{
+  "matched_files": [
+    { "uri": "file:///docs/database-configuration.docx", "unc": null, "score": 0.8432 }
+  ],
+  "num_matched_files": 1,
+  "chunks": [
+    {
+      "text": "To configure the database connection, set the DATABASE_URL environment variable...",
+      "score": 0.912,
+      "location": {
+        "uri": "file:///docs/setup-guide.pdf",
+        "unc": null,
+        "page": 12,
+        "line": null,
+        "heading_path": ["Getting Started", "Configuration", "Database"]
+      }
+    }
+  ],
+  "num_chunks": 1,
+  "query": "how to configure the database",
+  "retrieval_time": "0.42s"
+}
+```
+
+See [docs/tools.md](docs/tools.md) for every field, including how files on a
+mounted network share are presented.
+
+## Documentation
+
+- [docs/configuration.md](docs/configuration.md) — the config file, where it is discovered, and a reference for every setting.
+- [docs/cli.md](docs/cli.md) — `chunksilo` and `chunksilo-mcp` options, including `--check-files` for explaining why a file is or is not indexed.
+- [docs/tools.md](docs/tools.md) — the `search_docs` result shape.
+- [docs/mcp-clients.md](docs/mcp-clients.md) — client setup for Claude Code, Claude Desktop, Cline, Roo Code and opencode, over stdio or HTTP.
+- [docs/reverse-proxy.md](docs/reverse-proxy.md) — sharing an instance behind an authenticating reverse proxy, with oauth2-proxy and KeyCloak group-based access.
+- [docs/troubleshooting.md](docs/troubleshooting.md) — indexing surprises, network mounts, offline mode and more.
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

{chunksilo-2.3.4 → chunksilo-3.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chunksilo"
-version = "2.3.4"
+version = "3.0.0"
 description = "Local RAG-based semantic document search with MCP server interface"
 license = "Apache-2.0"
 requires-python = ">=3.11"
@@ -25,8 +25,11 @@ classifiers = [
 dynamic = ["dependencies"]
 
 [project.optional-dependencies]
-confluence = ["llama-index-readers-confluence>=0.6.0,<1"]
-jira = ["jira>=3.5.0,<4"]
+# Confluence and Jira support is now installed by default (see requirements.txt).
+# These extras are kept as empty aliases so existing `pip install chunksilo[confluence,jira]`
+# commands keep working without a warning.
+confluence = []
+jira = []
 test = ["pytest>=7.4.0,<9", "pytest-cov>=4.0", "requests>=2.31.0,<3"]
 
 [project.scripts]

{chunksilo-2.3.4 → chunksilo-3.0.0}/requirements.txt

@@ -5,8 +5,10 @@ llama-index-retrievers-bm25>=0.1.0,<1
 pillow>=10.3.0,<11
 pypdf>=5.1.0,<7
 python-docx>=1.1.0,<2
-mcp>=1.0.0,<2
+mcp>=1.10.0,<2
 huggingface-hub>=0.22.0,<2
 flashrank>=0.1.0,<1
 fastembed>=0.5.0,<1
 pyyaml>=6.0,<7
+llama-index-readers-confluence>=0.6.0,<1
+jira>=3.5.0,<4

{chunksilo-2.3.4 → chunksilo-3.0.0}/src/chunksilo/__init__.py

@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: Apache-2.0
 """ChunkSilo - Local RAG-based semantic document search."""
 
-__version__ = "2.3.4"
+__version__ = "3.0.0"