local-rag-mcp 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of local-rag-mcp might be problematic. Click here for more details.
- local_rag_mcp-2.1.0/LICENSE +21 -0
- local_rag_mcp-2.1.0/PKG-INFO +270 -0
- local_rag_mcp-2.1.0/README.md +230 -0
- local_rag_mcp-2.1.0/pyproject.toml +55 -0
- local_rag_mcp-2.1.0/setup.cfg +4 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/__init__.py +10 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/chunker.py +211 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/engine.py +613 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/lancedb_store.py +325 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/logger.py +133 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/parsers/__init__.py +42 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/parsers/excel_parser.py +108 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/parsers/pdf_parser.py +160 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/parsers/text_parser.py +98 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/parsers/word_parser.py +150 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/sampling.py +207 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/server.py +674 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/status.py +172 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp/watcher.py +244 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/PKG-INFO +270 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/SOURCES.txt +29 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/dependency_links.txt +1 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/entry_points.txt +2 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/requires.txt +15 -0
- local_rag_mcp-2.1.0/src/local_rag_mcp.egg-info/top_level.txt +1 -0
- local_rag_mcp-2.1.0/tests/test_chunker.py +126 -0
- local_rag_mcp-2.1.0/tests/test_engine.py +199 -0
- local_rag_mcp-2.1.0/tests/test_integration.py +142 -0
- local_rag_mcp-2.1.0/tests/test_lancedb_store.py +141 -0
- local_rag_mcp-2.1.0/tests/test_mcp_tools.py +124 -0
- local_rag_mcp-2.1.0/tests/test_parsers.py +178 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 RAG MCP Server Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: local-rag-mcp
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: RAG Knowledge MCP Server — FastMCP + LanceDB + Hybrid Search
|
|
5
|
+
Author-email: HeJuncheng <3410472132@qq.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/junchenghe01/local-rag-mcp
|
|
8
|
+
Project-URL: Repository, https://github.com/junchenghe01/local-rag-mcp
|
|
9
|
+
Project-URL: Issues, https://github.com/junchenghe01/local-rag-mcp/issues
|
|
10
|
+
Keywords: mcp,rag,lancedb,hybrid-search,llm,vector-database
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: mcp[cli]>=1.0
|
|
25
|
+
Requires-Dist: pydantic>=2.0
|
|
26
|
+
Requires-Dist: python-dotenv>=1.0
|
|
27
|
+
Requires-Dist: lancedb>=0.7
|
|
28
|
+
Requires-Dist: pyarrow>=16.0
|
|
29
|
+
Requires-Dist: watchdog>=4.0
|
|
30
|
+
Requires-Dist: loguru>=0.7
|
|
31
|
+
Requires-Dist: pypdf>=4.2
|
|
32
|
+
Requires-Dist: python-docx>=1.2
|
|
33
|
+
Requires-Dist: openpyxl>=3.1
|
|
34
|
+
Requires-Dist: pandas>=2.2
|
|
35
|
+
Requires-Dist: fastapi>=0.111
|
|
36
|
+
Requires-Dist: uvicorn[standard]>=0.29
|
|
37
|
+
Requires-Dist: aiofiles>=23.0
|
|
38
|
+
Requires-Dist: httpx>=0.27
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# Local RAG MCP Server v2.1.0
|
|
42
|
+
|
|
43
|
+
基于 FastMCP + LanceDB 的 RAG 知识库 MCP 服务器,完整实现 MCP 五大核心特征 (Tools / Resources / Prompts / Sampling / Roots)。
|
|
44
|
+
|
|
45
|
+
## V2.0 核心升级
|
|
46
|
+
|
|
47
|
+
| 特性 | V1.x | V2.0 |
|
|
48
|
+
|------|------|------|
|
|
49
|
+
| 向量存储 | llama_index VectorStoreIndex | **LanceDB** (Rust 核心, mmap, 无服务器) |
|
|
50
|
+
| 检索算法 | AutoMergingRetriever (纯语义) | **混合检索** (IVF-PQ + BM25 + RRF 重排) |
|
|
51
|
+
| 文档解析 | DoclingReader (黑盒) | **自研 ETL** (PyPDF/docx/pandas, 可控可采样) |
|
|
52
|
+
| 分块策略 | 256/512/1024 层级分块 | **智能分块** (600char + 句尾防截断) |
|
|
53
|
+
| 文件监控 | 启动时 manifest 比对 | **Watchdog 实时监控** + 1.0s 防抖 |
|
|
54
|
+
| 日志系统 | logging + RotatingFileHandler | **Loguru** (按天轮转 + zip 压缩) |
|
|
55
|
+
| MCP 工具 | 3 个 (health_check/status/query) | **7 个** (完整 CRUD + 混合检索) |
|
|
56
|
+
| MCP 特性 | 仅 Tools | **Tools + Resources + Prompts + Sampling + Roots** |
|
|
57
|
+
|
|
58
|
+
## 项目结构
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
src/local_rag_mcp/
|
|
62
|
+
├── __init__.py # 包标记
|
|
63
|
+
├── server.py # FastMCP 入口 — 7 工具 + 2 资源 + 1 提示词 + Sampling + Roots
|
|
64
|
+
├── engine.py # 核心引擎 — EmbeddingClient + HybridSearchEngine + IngestionPipeline
|
|
65
|
+
├── lancedb_store.py # LanceDB 存储层 (IVF-PQ + BM25 FTS)
|
|
66
|
+
├── chunker.py # 智能分块器 (600char/60overlap, 句尾防截断)
|
|
67
|
+
├── parsers/ # 多模态 ETL 解析器
|
|
68
|
+
│ ├── __init__.py # 解析器工厂 get_parser()
|
|
69
|
+
│ ├── pdf_parser.py # PyPDF 流式段落解析 + 高噪检测 → Sampling
|
|
70
|
+
│ ├── word_parser.py # python-docx 对象树遍历 + 表格序列化
|
|
71
|
+
│ ├── excel_parser.py # pandas 矩阵序列化 ("列A: 值A; 列B: 值B")
|
|
72
|
+
│ └── text_parser.py # MD/TXT/HTML 通用解析
|
|
73
|
+
├── watcher.py # Watchdog 文件监控 + 1.0s 防抖状态机
|
|
74
|
+
├── sampling.py # MCP Sampling 反向采样 (高噪文本清洗)
|
|
75
|
+
├── logger.py # Loguru 按天轮转 + zip 压缩 + 10 天保留
|
|
76
|
+
└── status.py # 线程安全服务状态追踪 (8 种状态)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## 快速开始
|
|
80
|
+
|
|
81
|
+
```powershell
|
|
82
|
+
# 1. 克隆项目,创建虚拟环境
|
|
83
|
+
python -m venv .venv
|
|
84
|
+
.\.venv\Scripts\python.exe -m pip install -e .
|
|
85
|
+
|
|
86
|
+
# 2. 配置 .env(Embedding 地址 + 文档路径)
|
|
87
|
+
# 编辑 .env,关键配置:
|
|
88
|
+
# EMBED_BASE_URL=http://localhost:11434
|
|
89
|
+
# EMBED_MODEL_NAME=qwen3-embedding:8b
|
|
90
|
+
# RAG_PROJECT_PATH=./documents
|
|
91
|
+
|
|
92
|
+
# 3. 启动
|
|
93
|
+
local-rag-mcp
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## 安装与运行
|
|
97
|
+
|
|
98
|
+
```powershell
|
|
99
|
+
# ---- 开发模式 (源码直接跑) ----
|
|
100
|
+
.venv\Scripts\python.exe -m src.local_rag_mcp.server
|
|
101
|
+
|
|
102
|
+
# ---- 生产模式 (PyPI 安装) ----
|
|
103
|
+
pip install local-rag-mcp
|
|
104
|
+
|
|
105
|
+
# 或本地构建 wheel 后安装
|
|
106
|
+
.venv\Scripts\python.exe -m build
|
|
107
|
+
pip install dist\local_rag_mcp-2.1.0-py3-none-any.whl
|
|
108
|
+
|
|
109
|
+
# 运行 (SSE 默认, 端口 8042)
|
|
110
|
+
local-rag-mcp
|
|
111
|
+
|
|
112
|
+
# 运行 (stdio 模式)
|
|
113
|
+
$env:MCP_TRANSPORT="stdio"; local-rag-mcp
|
|
114
|
+
|
|
115
|
+
# 卸载
|
|
116
|
+
pip uninstall local-rag-mcp -y
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## 接入 MCP 客户端
|
|
120
|
+
|
|
121
|
+
### SSE 模式(推荐)
|
|
122
|
+
|
|
123
|
+
先启动服务 `local-rag-mcp`,保持窗口打开:
|
|
124
|
+
|
|
125
|
+
```json
|
|
126
|
+
{
|
|
127
|
+
"mcpServers": {
|
|
128
|
+
"enterprise-local-rag": {
|
|
129
|
+
"type": "sse",
|
|
130
|
+
"url": "http://127.0.0.1:8042/sse"
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### stdio 模式(子进程,客户端自动拉起)
|
|
137
|
+
|
|
138
|
+
```json
|
|
139
|
+
{
|
|
140
|
+
"mcpServers": {
|
|
141
|
+
"enterprise-local-rag": {
|
|
142
|
+
"command": "local-rag-mcp",
|
|
143
|
+
"env": {
|
|
144
|
+
"MCP_TRANSPORT": "stdio",
|
|
145
|
+
"RAG_PROJECT_PATH": "./documents",
|
|
146
|
+
"EMBED_BASE_URL": "http://localhost:11434",
|
|
147
|
+
"EMBED_MODEL_NAME": "qwen3-embedding:8b"
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## 环境变量
|
|
155
|
+
|
|
156
|
+
| 变量 | 默认值 | 说明 |
|
|
157
|
+
|---|---|---|
|
|
158
|
+
| `RAG_PROJECT_PATH` | (必填) | 文档项目目录,启动时自动加载索引 + 文件监控 |
|
|
159
|
+
| `EMBED_BASE_URL` | `http://localhost:11434` | Ollama Embedding 服务地址 |
|
|
160
|
+
| `EMBED_MODEL_NAME` | `bge-m3` | Embedding 模型名称 (1024 维) |
|
|
161
|
+
| `EMBED_TIMEOUT` | `60.0` | Embedding 请求超时(秒) |
|
|
162
|
+
| `MCP_TRANSPORT` | `sse` | 传输模式: `sse` (守护进程) 或 `stdio` (子进程) |
|
|
163
|
+
| `MCP_HOST` | `127.0.0.1` | SSE 监听地址 |
|
|
164
|
+
| `MCP_PORT` | `8042` | SSE 监听端口 |
|
|
165
|
+
| `RAG_HYBRID_WEIGHT` | `0.7` | 混合检索权重 (0=纯BM25, 1=纯向量) |
|
|
166
|
+
| `RAG_CHUNK_SIZE` | `600` | 分块大小(字符数) |
|
|
167
|
+
| `RAG_CHUNK_OVERLAP` | `60` | 块间重叠(字符数) |
|
|
168
|
+
| `RAG_WATCH_DEBOUNCE` | `1.0` | 文件监控防抖间隔(秒) |
|
|
169
|
+
| `LOG_LEVEL` | `INFO` | 日志级别 |
|
|
170
|
+
|
|
171
|
+
## MCP 工具 (7)
|
|
172
|
+
|
|
173
|
+
| 工具 | 参数 | 说明 |
|
|
174
|
+
|---|---|---|
|
|
175
|
+
| `query_documents` | `query`, `limit?`, `scope?` | **混合检索** (密集向量 + BM25 + RRF),返回原始文档片段 |
|
|
176
|
+
| `read_chunk_neighbors` | `file_id`, `chunk_id`, `direction` | 读取相邻块 (prev/next/both),防止断章取义 |
|
|
177
|
+
| `ingest_file` | `path` | 索引单个文件 (PDF/Word/Excel/MD/TXT/HTML) |
|
|
178
|
+
| `ingest_data` | `content`, `metadata?` | 注入文本/Markdown 字符串到向量库 |
|
|
179
|
+
| `list_files` | 无 | 已索引文件清单及块统计 |
|
|
180
|
+
| `delete_file` | `file_id` | 物理擦除文件索引 + 重建 FTS |
|
|
181
|
+
| `status` | 无 | 系统健康诊断 (DB 统计 + Embedding + 文件监控) |
|
|
182
|
+
|
|
183
|
+
## MCP 资源 (2)
|
|
184
|
+
|
|
185
|
+
| URI | 说明 |
|
|
186
|
+
|---|---|
|
|
187
|
+
| `localrag://system/status` | 系统健康快照 (JSON): DB 统计 / Embedding / 文件监控 |
|
|
188
|
+
| `localrag://files/list` | 已索引文件清单 (JSON): 文件路径 + 块数量 |
|
|
189
|
+
|
|
190
|
+
## MCP 提示词 (1)
|
|
191
|
+
|
|
192
|
+
| 名称 | 参数 | 说明 |
|
|
193
|
+
|---|---|---|
|
|
194
|
+
| `review-codebase` | `focus_area` | 代码库评审 SOP 模板,自动注入 RAG 检索工作流 |
|
|
195
|
+
|
|
196
|
+
## RAG 管道 (V2.0)
|
|
197
|
+
|
|
198
|
+
1. **文件监控** — Watchdog 实时监控增删改,1.0s 防抖,自动触发 ETL
|
|
199
|
+
2. **多模态解析** — 自研解析器 (PyPDF/docx/pandas),PDF 高噪自动触发 Sampling 清洗
|
|
200
|
+
3. **智能分块** — 600 字符块 + 60 字符重叠 + 句尾标点语义切分
|
|
201
|
+
4. **向量化** — Ollama `bge-m3` (1024 维),直接调用 `/api/embeddings`
|
|
202
|
+
5. **双索引** — LanceDB IVF-PQ 密集向量索引 + BM25 稀疏全文倒排索引
|
|
203
|
+
6. **混合检索** — ANN + FTS 双路并行召回,RRF 无监督融合 (k=60, 权重可配置)
|
|
204
|
+
|
|
205
|
+
### RRF 融合公式
|
|
206
|
+
|
|
207
|
+
$$RRF\_Score(d) = \frac{w}{60 + R_{dense}(d)} + \frac{1 - w}{60 + R_{sparse}(d)}$$
|
|
208
|
+
|
|
209
|
+
- `w` = `RAG_HYBRID_WEIGHT` (默认 0.7)
|
|
210
|
+
- 当 w=0.7 时,兼顾深层语义并精确匹配代码/编号/专有名词
|
|
211
|
+
|
|
212
|
+
## 日志系统
|
|
213
|
+
|
|
214
|
+
### 控制台输出
|
|
215
|
+
彩色日志输出到 stderr,格式:`时间 | 级别 | 线程名 | 模块:行号 | 内容`
|
|
216
|
+
|
|
217
|
+
```
|
|
218
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.server:67 | Enterprise RAG MCP Server v2.0 启动
|
|
219
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.engine:85 | Embedding: bge-m3 @ http://localhost:11434 (dim=1024)
|
|
220
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.lancedb_store:68 | LanceDB 已连接 (.ragdb_lance/)
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### 文件归档
|
|
224
|
+
- **按天切分**: 每日 00:00 自动切分,文件名 `server_YYYY-MM-DD.log`
|
|
225
|
+
- **自动压缩**: 旧日志 zip 压缩
|
|
226
|
+
- **保留 10 天**: 超期日志自动物理擦除
|
|
227
|
+
- **全链路 Trace**: `[Trace] ▶ ...` / `[Trace] ✓ ... (XXms)` 毫秒级耗时审计
|
|
228
|
+
|
|
229
|
+
## 服务状态追踪
|
|
230
|
+
|
|
231
|
+
`status.py` 维护线程安全的全局状态单例,包含 8 种状态:
|
|
232
|
+
|
|
233
|
+
| 状态 | 说明 |
|
|
234
|
+
|---|---|
|
|
235
|
+
| `INITIALIZING` | 服务启动中 |
|
|
236
|
+
| `IDLE` | 空闲,等待请求 |
|
|
237
|
+
| `LOADING_DOCS` | 正在扫描/加载文档 |
|
|
238
|
+
| `INDEXING` | 正在构建向量索引 |
|
|
239
|
+
| `INGESTING` | 正在摄取文件/文本 |
|
|
240
|
+
| `QUERYING` | 正在执行混合检索 |
|
|
241
|
+
| `WATCHING` | 文件监控运行中 |
|
|
242
|
+
| `ERROR` | 发生错误(含错误信息) |
|
|
243
|
+
|
|
244
|
+
## Sampling 反向采样
|
|
245
|
+
|
|
246
|
+
当 PDF 解析器检测到高噪文本 (noise_level > 0.35) 时:
|
|
247
|
+
1. 后台服务挂起当前文件的写入事务
|
|
248
|
+
2. 通过 MCP `sampling/createMessage` 反向请求前台大模型进行文本清洗
|
|
249
|
+
3. 前台大模型完成推理清洗,返回结构化文本 + 核心关键词
|
|
250
|
+
4. 后台接收清洗结果,继续 ETL 流水线
|
|
251
|
+
|
|
252
|
+
**价值**: "白嫖"前台大模型的能力反哺本地知识库,本地服务保持轻量 (闲置物理内存 ≤ 35MB)。
|
|
253
|
+
|
|
254
|
+
## Roots 隐私隔离
|
|
255
|
+
|
|
256
|
+
当用户在编辑器中切换项目工作区时:
|
|
257
|
+
1. 客户端通过 `roots/list` 通知新的工作区路径
|
|
258
|
+
2. 服务器自动切换 LanceDB 命名空间,确保数据强隔离
|
|
259
|
+
3. 文件监控焦点无缝切换至新路径
|
|
260
|
+
4. 项目 A 的资产绝不会在项目 B 的对话中被意外召回
|
|
261
|
+
|
|
262
|
+
## 测试
|
|
263
|
+
|
|
264
|
+
```powershell
|
|
265
|
+
# 全部测试 (需要 Ollama 的会自动跳过)
|
|
266
|
+
.venv\Scripts\python.exe -m pytest tests/ -v
|
|
267
|
+
|
|
268
|
+
# 仅本地单元测试 (无需网络)
|
|
269
|
+
.venv\Scripts\python.exe -m pytest tests/ -v -m "not network"
|
|
270
|
+
```
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
# Local RAG MCP Server v2.1.0
|
|
2
|
+
|
|
3
|
+
基于 FastMCP + LanceDB 的 RAG 知识库 MCP 服务器,完整实现 MCP 五大核心特征 (Tools / Resources / Prompts / Sampling / Roots)。
|
|
4
|
+
|
|
5
|
+
## V2.0 核心升级
|
|
6
|
+
|
|
7
|
+
| 特性 | V1.x | V2.0 |
|
|
8
|
+
|------|------|------|
|
|
9
|
+
| 向量存储 | llama_index VectorStoreIndex | **LanceDB** (Rust 核心, mmap, 无服务器) |
|
|
10
|
+
| 检索算法 | AutoMergingRetriever (纯语义) | **混合检索** (IVF-PQ + BM25 + RRF 重排) |
|
|
11
|
+
| 文档解析 | DoclingReader (黑盒) | **自研 ETL** (PyPDF/docx/pandas, 可控可采样) |
|
|
12
|
+
| 分块策略 | 256/512/1024 层级分块 | **智能分块** (600char + 句尾防截断) |
|
|
13
|
+
| 文件监控 | 启动时 manifest 比对 | **Watchdog 实时监控** + 1.0s 防抖 |
|
|
14
|
+
| 日志系统 | logging + RotatingFileHandler | **Loguru** (按天轮转 + zip 压缩) |
|
|
15
|
+
| MCP 工具 | 3 个 (health_check/status/query) | **7 个** (完整 CRUD + 混合检索) |
|
|
16
|
+
| MCP 特性 | 仅 Tools | **Tools + Resources + Prompts + Sampling + Roots** |
|
|
17
|
+
|
|
18
|
+
## 项目结构
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
src/local_rag_mcp/
|
|
22
|
+
├── __init__.py # 包标记
|
|
23
|
+
├── server.py # FastMCP 入口 — 7 工具 + 2 资源 + 1 提示词 + Sampling + Roots
|
|
24
|
+
├── engine.py # 核心引擎 — EmbeddingClient + HybridSearchEngine + IngestionPipeline
|
|
25
|
+
├── lancedb_store.py # LanceDB 存储层 (IVF-PQ + BM25 FTS)
|
|
26
|
+
├── chunker.py # 智能分块器 (600char/60overlap, 句尾防截断)
|
|
27
|
+
├── parsers/ # 多模态 ETL 解析器
|
|
28
|
+
│ ├── __init__.py # 解析器工厂 get_parser()
|
|
29
|
+
│ ├── pdf_parser.py # PyPDF 流式段落解析 + 高噪检测 → Sampling
|
|
30
|
+
│ ├── word_parser.py # python-docx 对象树遍历 + 表格序列化
|
|
31
|
+
│ ├── excel_parser.py # pandas 矩阵序列化 ("列A: 值A; 列B: 值B")
|
|
32
|
+
│ └── text_parser.py # MD/TXT/HTML 通用解析
|
|
33
|
+
├── watcher.py # Watchdog 文件监控 + 1.0s 防抖状态机
|
|
34
|
+
├── sampling.py # MCP Sampling 反向采样 (高噪文本清洗)
|
|
35
|
+
├── logger.py # Loguru 按天轮转 + zip 压缩 + 10 天保留
|
|
36
|
+
└── status.py # 线程安全服务状态追踪 (8 种状态)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## 快速开始
|
|
40
|
+
|
|
41
|
+
```powershell
|
|
42
|
+
# 1. 克隆项目,创建虚拟环境
|
|
43
|
+
python -m venv .venv
|
|
44
|
+
.\.venv\Scripts\python.exe -m pip install -e .
|
|
45
|
+
|
|
46
|
+
# 2. 配置 .env(Embedding 地址 + 文档路径)
|
|
47
|
+
# 编辑 .env,关键配置:
|
|
48
|
+
# EMBED_BASE_URL=http://localhost:11434
|
|
49
|
+
# EMBED_MODEL_NAME=qwen3-embedding:8b
|
|
50
|
+
# RAG_PROJECT_PATH=./documents
|
|
51
|
+
|
|
52
|
+
# 3. 启动
|
|
53
|
+
local-rag-mcp
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## 安装与运行
|
|
57
|
+
|
|
58
|
+
```powershell
|
|
59
|
+
# ---- 开发模式 (源码直接跑) ----
|
|
60
|
+
.venv\Scripts\python.exe -m src.local_rag_mcp.server
|
|
61
|
+
|
|
62
|
+
# ---- 生产模式 (PyPI 安装) ----
|
|
63
|
+
pip install local-rag-mcp
|
|
64
|
+
|
|
65
|
+
# 或本地构建 wheel 后安装
|
|
66
|
+
.venv\Scripts\python.exe -m build
|
|
67
|
+
pip install dist\local_rag_mcp-2.1.0-py3-none-any.whl
|
|
68
|
+
|
|
69
|
+
# 运行 (SSE 默认, 端口 8042)
|
|
70
|
+
local-rag-mcp
|
|
71
|
+
|
|
72
|
+
# 运行 (stdio 模式)
|
|
73
|
+
$env:MCP_TRANSPORT="stdio"; local-rag-mcp
|
|
74
|
+
|
|
75
|
+
# 卸载
|
|
76
|
+
pip uninstall local-rag-mcp -y
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## 接入 MCP 客户端
|
|
80
|
+
|
|
81
|
+
### SSE 模式(推荐)
|
|
82
|
+
|
|
83
|
+
先启动服务 `local-rag-mcp`,保持窗口打开:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{
|
|
87
|
+
"mcpServers": {
|
|
88
|
+
"enterprise-local-rag": {
|
|
89
|
+
"type": "sse",
|
|
90
|
+
"url": "http://127.0.0.1:8042/sse"
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### stdio 模式(子进程,客户端自动拉起)
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{
|
|
100
|
+
"mcpServers": {
|
|
101
|
+
"enterprise-local-rag": {
|
|
102
|
+
"command": "local-rag-mcp",
|
|
103
|
+
"env": {
|
|
104
|
+
"MCP_TRANSPORT": "stdio",
|
|
105
|
+
"RAG_PROJECT_PATH": "./documents",
|
|
106
|
+
"EMBED_BASE_URL": "http://localhost:11434",
|
|
107
|
+
"EMBED_MODEL_NAME": "qwen3-embedding:8b"
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## 环境变量
|
|
115
|
+
|
|
116
|
+
| 变量 | 默认值 | 说明 |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| `RAG_PROJECT_PATH` | (必填) | 文档项目目录,启动时自动加载索引 + 文件监控 |
|
|
119
|
+
| `EMBED_BASE_URL` | `http://localhost:11434` | Ollama Embedding 服务地址 |
|
|
120
|
+
| `EMBED_MODEL_NAME` | `bge-m3` | Embedding 模型名称 (1024 维) |
|
|
121
|
+
| `EMBED_TIMEOUT` | `60.0` | Embedding 请求超时(秒) |
|
|
122
|
+
| `MCP_TRANSPORT` | `sse` | 传输模式: `sse` (守护进程) 或 `stdio` (子进程) |
|
|
123
|
+
| `MCP_HOST` | `127.0.0.1` | SSE 监听地址 |
|
|
124
|
+
| `MCP_PORT` | `8042` | SSE 监听端口 |
|
|
125
|
+
| `RAG_HYBRID_WEIGHT` | `0.7` | 混合检索权重 (0=纯BM25, 1=纯向量) |
|
|
126
|
+
| `RAG_CHUNK_SIZE` | `600` | 分块大小(字符数) |
|
|
127
|
+
| `RAG_CHUNK_OVERLAP` | `60` | 块间重叠(字符数) |
|
|
128
|
+
| `RAG_WATCH_DEBOUNCE` | `1.0` | 文件监控防抖间隔(秒) |
|
|
129
|
+
| `LOG_LEVEL` | `INFO` | 日志级别 |
|
|
130
|
+
|
|
131
|
+
## MCP 工具 (7)
|
|
132
|
+
|
|
133
|
+
| 工具 | 参数 | 说明 |
|
|
134
|
+
|---|---|---|
|
|
135
|
+
| `query_documents` | `query`, `limit?`, `scope?` | **混合检索** (密集向量 + BM25 + RRF),返回原始文档片段 |
|
|
136
|
+
| `read_chunk_neighbors` | `file_id`, `chunk_id`, `direction` | 读取相邻块 (prev/next/both),防止断章取义 |
|
|
137
|
+
| `ingest_file` | `path` | 索引单个文件 (PDF/Word/Excel/MD/TXT/HTML) |
|
|
138
|
+
| `ingest_data` | `content`, `metadata?` | 注入文本/Markdown 字符串到向量库 |
|
|
139
|
+
| `list_files` | 无 | 已索引文件清单及块统计 |
|
|
140
|
+
| `delete_file` | `file_id` | 物理擦除文件索引 + 重建 FTS |
|
|
141
|
+
| `status` | 无 | 系统健康诊断 (DB 统计 + Embedding + 文件监控) |
|
|
142
|
+
|
|
143
|
+
## MCP 资源 (2)
|
|
144
|
+
|
|
145
|
+
| URI | 说明 |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `localrag://system/status` | 系统健康快照 (JSON): DB 统计 / Embedding / 文件监控 |
|
|
148
|
+
| `localrag://files/list` | 已索引文件清单 (JSON): 文件路径 + 块数量 |
|
|
149
|
+
|
|
150
|
+
## MCP 提示词 (1)
|
|
151
|
+
|
|
152
|
+
| 名称 | 参数 | 说明 |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| `review-codebase` | `focus_area` | 代码库评审 SOP 模板,自动注入 RAG 检索工作流 |
|
|
155
|
+
|
|
156
|
+
## RAG 管道 (V2.0)
|
|
157
|
+
|
|
158
|
+
1. **文件监控** — Watchdog 实时监控增删改,1.0s 防抖,自动触发 ETL
|
|
159
|
+
2. **多模态解析** — 自研解析器 (PyPDF/docx/pandas),PDF 高噪自动触发 Sampling 清洗
|
|
160
|
+
3. **智能分块** — 600 字符块 + 60 字符重叠 + 句尾标点语义切分
|
|
161
|
+
4. **向量化** — Ollama `bge-m3` (1024 维),直接调用 `/api/embeddings`
|
|
162
|
+
5. **双索引** — LanceDB IVF-PQ 密集向量索引 + BM25 稀疏全文倒排索引
|
|
163
|
+
6. **混合检索** — ANN + FTS 双路并行召回,RRF 无监督融合 (k=60, 权重可配置)
|
|
164
|
+
|
|
165
|
+
### RRF 融合公式
|
|
166
|
+
|
|
167
|
+
$$RRF\_Score(d) = \frac{w}{60 + R_{dense}(d)} + \frac{1 - w}{60 + R_{sparse}(d)}$$
|
|
168
|
+
|
|
169
|
+
- `w` = `RAG_HYBRID_WEIGHT` (默认 0.7)
|
|
170
|
+
- 当 w=0.7 时,兼顾深层语义并精确匹配代码/编号/专有名词
|
|
171
|
+
|
|
172
|
+
## 日志系统
|
|
173
|
+
|
|
174
|
+
### 控制台输出
|
|
175
|
+
彩色日志输出到 stderr,格式:`时间 | 级别 | 线程名 | 模块:行号 | 内容`
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.server:67 | Enterprise RAG MCP Server v2.0 启动
|
|
179
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.engine:85 | Embedding: bge-m3 @ http://localhost:11434 (dim=1024)
|
|
180
|
+
20:53:55 | INFO | MainThread | local_rag_mcp.lancedb_store:68 | LanceDB 已连接 (.ragdb_lance/)
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### 文件归档
|
|
184
|
+
- **按天切分**: 每日 00:00 自动切分,文件名 `server_YYYY-MM-DD.log`
|
|
185
|
+
- **自动压缩**: 旧日志 zip 压缩
|
|
186
|
+
- **保留 10 天**: 超期日志自动物理擦除
|
|
187
|
+
- **全链路 Trace**: `[Trace] ▶ ...` / `[Trace] ✓ ... (XXms)` 毫秒级耗时审计
|
|
188
|
+
|
|
189
|
+
## 服务状态追踪
|
|
190
|
+
|
|
191
|
+
`status.py` 维护线程安全的全局状态单例,包含 8 种状态:
|
|
192
|
+
|
|
193
|
+
| 状态 | 说明 |
|
|
194
|
+
|---|---|
|
|
195
|
+
| `INITIALIZING` | 服务启动中 |
|
|
196
|
+
| `IDLE` | 空闲,等待请求 |
|
|
197
|
+
| `LOADING_DOCS` | 正在扫描/加载文档 |
|
|
198
|
+
| `INDEXING` | 正在构建向量索引 |
|
|
199
|
+
| `INGESTING` | 正在摄取文件/文本 |
|
|
200
|
+
| `QUERYING` | 正在执行混合检索 |
|
|
201
|
+
| `WATCHING` | 文件监控运行中 |
|
|
202
|
+
| `ERROR` | 发生错误(含错误信息) |
|
|
203
|
+
|
|
204
|
+
## Sampling 反向采样
|
|
205
|
+
|
|
206
|
+
当 PDF 解析器检测到高噪文本 (noise_level > 0.35) 时:
|
|
207
|
+
1. 后台服务挂起当前文件的写入事务
|
|
208
|
+
2. 通过 MCP `sampling/createMessage` 反向请求前台大模型进行文本清洗
|
|
209
|
+
3. 前台大模型完成推理清洗,返回结构化文本 + 核心关键词
|
|
210
|
+
4. 后台接收清洗结果,继续 ETL 流水线
|
|
211
|
+
|
|
212
|
+
**价值**: "白嫖"前台大模型的能力反哺本地知识库,本地服务保持轻量 (闲置物理内存 ≤ 35MB)。
|
|
213
|
+
|
|
214
|
+
## Roots 隐私隔离
|
|
215
|
+
|
|
216
|
+
当用户在编辑器中切换项目工作区时:
|
|
217
|
+
1. 客户端通过 `roots/list` 通知新的工作区路径
|
|
218
|
+
2. 服务器自动切换 LanceDB 命名空间,确保数据强隔离
|
|
219
|
+
3. 文件监控焦点无缝切换至新路径
|
|
220
|
+
4. 项目 A 的资产绝不会在项目 B 的对话中被意外召回
|
|
221
|
+
|
|
222
|
+
## 测试
|
|
223
|
+
|
|
224
|
+
```powershell
|
|
225
|
+
# 全部测试 (需要 Ollama 的会自动跳过)
|
|
226
|
+
.venv\Scripts\python.exe -m pytest tests/ -v
|
|
227
|
+
|
|
228
|
+
# 仅本地单元测试 (无需网络)
|
|
229
|
+
.venv\Scripts\python.exe -m pytest tests/ -v -m "not network"
|
|
230
|
+
```
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=75"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "local-rag-mcp"
|
|
7
|
+
version = "2.1.0"
|
|
8
|
+
description = "RAG Knowledge MCP Server — FastMCP + LanceDB + Hybrid Search"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "HeJuncheng", email = "3410472132@qq.com"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["mcp", "rag", "lancedb", "hybrid-search", "llm", "vector-database"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"mcp[cli]>=1.0",
|
|
30
|
+
"pydantic>=2.0",
|
|
31
|
+
"python-dotenv>=1.0",
|
|
32
|
+
"lancedb>=0.7",
|
|
33
|
+
"pyarrow>=16.0",
|
|
34
|
+
"watchdog>=4.0",
|
|
35
|
+
"loguru>=0.7",
|
|
36
|
+
"pypdf>=4.2",
|
|
37
|
+
"python-docx>=1.2",
|
|
38
|
+
"openpyxl>=3.1",
|
|
39
|
+
"pandas>=2.2",
|
|
40
|
+
"fastapi>=0.111",
|
|
41
|
+
"uvicorn[standard]>=0.29",
|
|
42
|
+
"aiofiles>=23.0",
|
|
43
|
+
"httpx>=0.27",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.urls]
|
|
47
|
+
Homepage = "https://github.com/junchenghe01/local-rag-mcp"
|
|
48
|
+
Repository = "https://github.com/junchenghe01/local-rag-mcp"
|
|
49
|
+
Issues = "https://github.com/junchenghe01/local-rag-mcp/issues"
|
|
50
|
+
|
|
51
|
+
[project.scripts]
|
|
52
|
+
local-rag-mcp = "local_rag_mcp.server:main"
|
|
53
|
+
|
|
54
|
+
[tool.setuptools.packages.find]
|
|
55
|
+
where = ["src"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""RAG MCP Server V2.0 — 基于 FastMCP + LanceDB 的企业级文档检索引擎。
|
|
2
|
+
|
|
3
|
+
完整实现 MCP 五大核心特征:
|
|
4
|
+
- Tools (7): ingest_file, ingest_data, query_documents, read_chunk_neighbors,
|
|
5
|
+
list_files, delete_file, status
|
|
6
|
+
- Resources (2): localrag://system/status, localrag://files/list
|
|
7
|
+
- Prompts (1): review-codebase
|
|
8
|
+
- Sampling: PDF 高噪文本反向清洗
|
|
9
|
+
- Roots: 动态工作区隔离
|
|
10
|
+
"""
|