modular-rag 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modular_rag-0.1.0.dist-info/METADATA +1073 -0
- modular_rag-0.1.0.dist-info/RECORD +58 -0
- modular_rag-0.1.0.dist-info/WHEEL +5 -0
- modular_rag-0.1.0.dist-info/licenses/LICENSE +21 -0
- modular_rag-0.1.0.dist-info/top_level.txt +1 -0
- ragkit/__init__.py +53 -0
- ragkit/callbacks/__init__.py +20 -0
- ragkit/callbacks/builtin.py +259 -0
- ragkit/callbacks/events.py +58 -0
- ragkit/callbacks/manager.py +89 -0
- ragkit/callbacks/progress.py +57 -0
- ragkit/catalog/__init__.py +16 -0
- ragkit/catalog/base.py +74 -0
- ragkit/catalog/chunk_store.py +109 -0
- ragkit/catalog/filtering.py +61 -0
- ragkit/catalog/models.py +236 -0
- ragkit/catalog/scalar.py +122 -0
- ragkit/catalog/store_catalog.py +66 -0
- ragkit/catalog/version.py +50 -0
- ragkit/core/__init__.py +11 -0
- ragkit/core/config.py +60 -0
- ragkit/core/embeddings.py +161 -0
- ragkit/core/filtering.py +44 -0
- ragkit/core/hashing.py +60 -0
- ragkit/core/schema.py +67 -0
- ragkit/middleware/__init__.py +20 -0
- ragkit/middleware/base.py +117 -0
- ragkit/middleware/builtin.py +256 -0
- ragkit/pipeline.py +391 -0
- ragkit/py.typed +0 -0
- ragkit/ragkit_version.py +1 -0
- ragkit/readers/__init__.py +61 -0
- ragkit/readers/_common.py +38 -0
- ragkit/readers/base.py +35 -0
- ragkit/readers/markdown_parser.py +221 -0
- ragkit/readers/office.py +291 -0
- ragkit/readers/pdf.py +162 -0
- ragkit/readers/text.py +62 -0
- ragkit/rerank/__init__.py +95 -0
- ragkit/rerank/base.py +63 -0
- ragkit/retrieval/__init__.py +7 -0
- ragkit/retrieval/fusion.py +61 -0
- ragkit/retrieval/hybrid.py +81 -0
- ragkit/retrieval/retrievers.py +133 -0
- ragkit/splitters/__init__.py +41 -0
- ragkit/splitters/base.py +83 -0
- ragkit/splitters/heading.py +76 -0
- ragkit/splitters/qa.py +26 -0
- ragkit/splitters/recursive.py +49 -0
- ragkit/splitters/table.py +54 -0
- ragkit/store/__init__.py +23 -0
- ragkit/store/base.py +40 -0
- ragkit/store/memory.py +55 -0
- ragkit/store/qdrant_store.py +203 -0
- ragkit/store/weaviate_store.py +249 -0
- ragkit/utils/__init__.py +5 -0
- ragkit/utils/registry.py +46 -0
- ragkit/utils/timeout.py +49 -0
|
@@ -0,0 +1,1073 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: modular-rag
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 可扩展、可替换抽象的 RAG 库:多格式读取、结构化切分、RRF 混合召回、主表+附表多版本、回调与中间件
|
|
5
|
+
Author: ragkit contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: rag,retrieval,vector-database,embeddings,rrf,chunking,llm
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Typing :: Typed
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Provides-Extra: pdf
|
|
22
|
+
Requires-Dist: pymupdf>=1.23; extra == "pdf"
|
|
23
|
+
Provides-Extra: word
|
|
24
|
+
Requires-Dist: python-docx>=1.0; extra == "word"
|
|
25
|
+
Provides-Extra: ppt
|
|
26
|
+
Requires-Dist: python-pptx>=0.6; extra == "ppt"
|
|
27
|
+
Provides-Extra: excel
|
|
28
|
+
Requires-Dist: openpyxl>=3.1; extra == "excel"
|
|
29
|
+
Provides-Extra: rtf
|
|
30
|
+
Requires-Dist: striprtf>=0.0.26; extra == "rtf"
|
|
31
|
+
Provides-Extra: rerank
|
|
32
|
+
Requires-Dist: sentence-transformers>=2.2; extra == "rerank"
|
|
33
|
+
Provides-Extra: qdrant
|
|
34
|
+
Requires-Dist: qdrant-client>=1.7; extra == "qdrant"
|
|
35
|
+
Provides-Extra: weaviate
|
|
36
|
+
Requires-Dist: weaviate-client>=4.4; extra == "weaviate"
|
|
37
|
+
Provides-Extra: office
|
|
38
|
+
Requires-Dist: python-docx>=1.0; extra == "office"
|
|
39
|
+
Requires-Dist: python-pptx>=0.6; extra == "office"
|
|
40
|
+
Requires-Dist: openpyxl>=3.1; extra == "office"
|
|
41
|
+
Provides-Extra: all
|
|
42
|
+
Requires-Dist: pymupdf>=1.23; extra == "all"
|
|
43
|
+
Requires-Dist: python-docx>=1.0; extra == "all"
|
|
44
|
+
Requires-Dist: python-pptx>=0.6; extra == "all"
|
|
45
|
+
Requires-Dist: openpyxl>=3.1; extra == "all"
|
|
46
|
+
Requires-Dist: striprtf>=0.0.26; extra == "all"
|
|
47
|
+
Requires-Dist: sentence-transformers>=2.2; extra == "all"
|
|
48
|
+
Requires-Dist: qdrant-client>=1.7; extra == "all"
|
|
49
|
+
Requires-Dist: weaviate-client>=4.4; extra == "all"
|
|
50
|
+
Provides-Extra: dev
|
|
51
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
52
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
53
|
+
Requires-Dist: twine>=4.0; extra == "dev"
|
|
54
|
+
Dynamic: license-file
|
|
55
|
+
|
|
56
|
+
# ragkit
|
|
57
|
+
|
|
58
|
+
一个**可扩展、可替换抽象**的 RAG 库。核心零第三方依赖即可运行,PDF / 模型重排等重依赖按需安装、延迟导入。
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
from ragkit import RAGPipeline, PipelineConfig
|
|
62
|
+
|
|
63
|
+
pipe = RAGPipeline(PipelineConfig())
|
|
64
|
+
pipe.ingest("docs/guide.md")
|
|
65
|
+
for r in pipe.retrieve("超时设置默认值是多少?"):
|
|
66
|
+
print(r.rank, r.score, r.chunk.title_path, r.chunk.text[:60])
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## 目录
|
|
72
|
+
|
|
73
|
+
- [特性](#特性)
|
|
74
|
+
- [安装](#安装)
|
|
75
|
+
- [快速开始](#快速开始)
|
|
76
|
+
- [核心概念](#核心概念)
|
|
77
|
+
- [使用指南](#使用指南)
|
|
78
|
+
- [1. 数据读取](#1-数据读取)
|
|
79
|
+
- [2. 切分](#2-切分)
|
|
80
|
+
- [3. 嵌入与向量库](#3-嵌入与向量库)
|
|
81
|
+
- [4. 召回](#4-召回)
|
|
82
|
+
- [5. 重排](#5-重排)
|
|
83
|
+
- [6. 主表 + 附表(多版本)](#6-主表--附表多版本)
|
|
84
|
+
- [7. 版本与权限](#7-版本与权限)
|
|
85
|
+
- [8. 回调与进度](#8-回调与进度)
|
|
86
|
+
- [9. 中间件](#9-中间件)
|
|
87
|
+
- [扩展指南](#扩展指南)
|
|
88
|
+
- [配置参考](#配置参考)
|
|
89
|
+
- [目录结构](#目录结构)
|
|
90
|
+
- [测试](#测试)
|
|
91
|
+
- [注意事项](#注意事项)
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## 特性
|
|
96
|
+
|
|
97
|
+
| 能力 | 说明 |
|
|
98
|
+
|---|---|
|
|
99
|
+
| **多格式读取** | Markdown / PDF / Word / PPT / Excel / CSV / HTML / EPUB / RTF / JSON,全部走基类扩展 |
|
|
100
|
+
| **结构化切分** | 递归 / 标题层级(保留完整路径)/ 表格(表头+行)/ 问答对;块长与重叠可配 |
|
|
101
|
+
| **自己实现的 RRF** | 多路召回融合,公式 `Σ wᵢ/(k+rankᵢ)`,不依赖向量库 |
|
|
102
|
+
| **可配置召回** | 默认混合(向量 + 关键词),可切单路;权重、候选数、top_k 可调 |
|
|
103
|
+
| **多方式重排** | 规则 / CrossEncoder / LLM,统一超时兜底 |
|
|
104
|
+
| **主表 + 附表** | 文件级与分段级两张表,同业务 id 多版本自动取最高版本 |
|
|
105
|
+
| **三套向量库** | 内存 / Qdrant / Weaviate,同一接口互换 |
|
|
106
|
+
| **可观测** | 生命周期事件、实时进度、错误上报、指标、审计、告警 |
|
|
107
|
+
| **可干预** | 中间件可改 query、调参数、过滤结果、短路、重试、熔断 |
|
|
108
|
+
| **零依赖可跑** | 核心仅标准库;重依赖延迟导入并给出明确安装提示 |
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## 安装
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
# 从 PyPI(包名 modular-rag,import 名仍是 ragkit)
|
|
116
|
+
pip install modular-rag
|
|
117
|
+
pip install "modular-rag[all]" # 含全部可选依赖
|
|
118
|
+
|
|
119
|
+
# 本地开发(可编辑安装)
|
|
120
|
+
pip install -e .
|
|
121
|
+
pip install -e ".[dev]" # 含 pytest / build / twine
|
|
122
|
+
|
|
123
|
+
# 从源码构建并安装
|
|
124
|
+
python -m build # 产出 dist/modular_rag-0.1.0-py3-none-any.whl 与 .tar.gz
|
|
125
|
+
pip install dist/modular_rag-0.1.0-py3-none-any.whl
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
**可选依赖分组**:`pdf` / `word` / `ppt` / `excel` / `office` / `rtf` / `rerank` / `qdrant` / `weaviate` / `all` / `dev`。
|
|
129
|
+
|
|
130
|
+
也可以单独安装:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
pip install pymupdf # PDF(去页眉页脚/目录、表格、扫描件)
|
|
134
|
+
pip install python-docx # Word .docx
|
|
135
|
+
pip install python-pptx # PowerPoint
|
|
136
|
+
pip install openpyxl # Excel
|
|
137
|
+
pip install sentence-transformers # CrossEncoder 重排
|
|
138
|
+
pip install qdrant-client # Qdrant 后端
|
|
139
|
+
pip install weaviate-client # Weaviate 后端
|
|
140
|
+
pip install striprtf # 更完整的 RTF 解析(可选)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
> 不装任何可选依赖也能跑通核心流程:读取 txt/md/json/csv/html/epub、切分、内存向量库、混合召回、规则重排。
|
|
144
|
+
|
|
145
|
+
> OCR 不内置,通过 `ocr_fn(page_index, png_bytes) -> str` 钩子注入,可接 pytesseract。
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
## 快速开始
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from ragkit import RAGPipeline, PipelineConfig, SplitConfig, RetrievalConfig
|
|
153
|
+
|
|
154
|
+
pipe = RAGPipeline(PipelineConfig(
|
|
155
|
+
split=SplitConfig(chunk_size=800, chunk_overlap=120),
|
|
156
|
+
retrieval=RetrievalConfig(mode="hybrid", top_k=10),
|
|
157
|
+
))
|
|
158
|
+
|
|
159
|
+
pipe.ingest("docs/") # 目录、文件、Document 列表都支持
|
|
160
|
+
results = pipe.retrieve("超时默认值")
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
一次入库多种来源:
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
pipe.ingest(["a.md", "b.pdf", "c.docx"])
|
|
167
|
+
pipe.ingest(Document(text="# 标题\n\n内容", metadata={"filter": {"doc_type": "markdown"}}))
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
---
|
|
171
|
+
|
|
172
|
+
## 核心概念
|
|
173
|
+
|
|
174
|
+
**数据流**
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
原始文件 → Reader → Document → Splitter → Chunk → Embedder → VectorStore
|
|
178
|
+
↓
|
|
179
|
+
查询文本 → Retriever(向量 + 关键词)→ RRF 融合 → Reranker → 结果
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
**元数据四类**(贯穿全库)
|
|
183
|
+
|
|
184
|
+
| 类别 | 字段 |
|
|
185
|
+
|---|---|
|
|
186
|
+
| `locator` 定位 | `doc_id` / `title_path` / `section` / `page_no` / `chunk_index` / `prev_chunk_id` / `next_chunk_id` |
|
|
187
|
+
| `filter` 过滤 | `business_id` / `version` / `status` / `owner` / `md5` / `doc_type` / `language` / `acl` |
|
|
188
|
+
| `evidence` 证据链 | `model_id` / `dimension` / `normalization` / `chunk_params` / `imported_at` |
|
|
189
|
+
| `provenance` 溯源 | `source_uri` / `parse_flags` / `front_matter` |
|
|
190
|
+
|
|
191
|
+
**这些字段由谁填?** 骨架全部由库自动创建和填充,使用者只需提供业务字段:
|
|
192
|
+
|
|
193
|
+
| 字段 | 谁填 | 何时 |
|
|
194
|
+
|---|---|---|
|
|
195
|
+
| `metadata` 四类骨架 | 库 | `Document` 创建时初始化 |
|
|
196
|
+
| `title_path` / `chunk_index` / 邻接 | Splitter | 切分时 |
|
|
197
|
+
| `page_no` | Reader(PDF/PPT) | 解析时 |
|
|
198
|
+
| `doc_type` | Reader 路由 | 读取时 |
|
|
199
|
+
| `source_uri` / `parse_flags` / `front_matter` | Reader | 解析时 |
|
|
200
|
+
| `model_id` / `dimension` / `normalization` / `chunk_params` / `imported_at` | Pipeline | 入库时 |
|
|
201
|
+
| `business_id` / `version` / `status` / `owner` / `md5` | 主表 `FileRecord` | 传 `record=` 时(`md5` 未传则自动计算) |
|
|
202
|
+
| **自定义扩展字段** | **使用者** | 见下 |
|
|
203
|
+
|
|
204
|
+
**扩展字段怎么赋值**
|
|
205
|
+
|
|
206
|
+
方式一:通过主表记录(推荐,随版本管理)
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
@dataclass
|
|
210
|
+
class MyFileRecord(FileRecord):
|
|
211
|
+
department: str = ""
|
|
212
|
+
|
|
213
|
+
pipe.ingest(path, record=MyFileRecord("guide", "v1", department="研发"))
|
|
214
|
+
|
|
215
|
+
# 或者动态挂
|
|
216
|
+
rec = FileRecord("guide", "v1")
|
|
217
|
+
rec.set_extra("tenant", "acme")
|
|
218
|
+
pipe.ingest(path, record=rec)
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
方式二:`ingest(metadata=...)`,不必构造 Document(适合批量场景)
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
pipe.ingest(path, metadata={"department": "研发", "tenant": "acme"})
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
- 传入的**普通键**自动写入 `filter`,可直接参与前置过滤
|
|
228
|
+
- 传入 `locator` / `filter` / `evidence` / `provenance` 子字典则按键合并,不覆盖其它字段
|
|
229
|
+
- 优先级最高,可覆盖框架默认值(如自定义 `page_no`)
|
|
230
|
+
|
|
231
|
+
方式三:构造 `Document` 时直接写 metadata(适合自定义 Reader)
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
Document(text=..., metadata={"filter": {"department": "研发", "acl": []}})
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
三种方式最终都落到同一处:`chunk.metadata["filter"]`,因此都能用于检索过滤:
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
pipe.retrieve("超时", filter_fn=lambda m: m["filter"].get("department") == "研发")
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
落库时,非标准键会自动收进 `ChunkRecord.extra`,取回时再合并回 filter,不丢字段。
|
|
244
|
+
|
|
245
|
+
**三条扩展路径**
|
|
246
|
+
|
|
247
|
+
| 方式 | 适用 | 特点 |
|
|
248
|
+
|---|---|---|
|
|
249
|
+
| 注册表 | 新增内置没有的实现 | `@READERS.register("xxx")`,可按名创建 |
|
|
250
|
+
| 构造注入 | 局部替换某个实现 | `RAGPipeline(reader=MyReader())`,不影响全局 |
|
|
251
|
+
| 中间件 | 干预流程 | 改参数、拦结果、短路、重试 |
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## 使用指南
|
|
256
|
+
|
|
257
|
+
### 1. 数据读取
|
|
258
|
+
|
|
259
|
+
| 格式 | 后缀 | 依赖 | 说明 |
|
|
260
|
+
|---|---|---|---|
|
|
261
|
+
| 纯文本 | `.txt` `.log` | 无 | 直接读取 |
|
|
262
|
+
| Markdown | `.md` `.markdown` | 无 | front matter→元数据;代码块/表格原子切分 |
|
|
263
|
+
| JSON 问答 | `.json` `.jsonl` | 无 | `q/a` 或 `question/answer` |
|
|
264
|
+
| CSV / TSV | `.csv` `.tsv` | 无 | 转 Markdown 表格 |
|
|
265
|
+
| HTML | `.html` `.htm` | 无 | 提取正文,`h1-h6`→标题,忽略 script/style |
|
|
266
|
+
| EPUB | `.epub` | 无 | 每章一个 Document |
|
|
267
|
+
| RTF | `.rtf` | 可选 striprtf | 无依赖时内置正则兜底 |
|
|
268
|
+
| PDF | `.pdf` | pymupdf | 去页眉页脚/目录、表格提取、扫描件 OCR 钩子 |
|
|
269
|
+
| Word | `.docx` `.docm` | python-docx | 保持段落+表格顺序,标题样式→Markdown |
|
|
270
|
+
| Word 旧版 | `.doc` | antiword/libreoffice | 外部转换 |
|
|
271
|
+
| PowerPoint | `.pptx` | python-pptx | 每页一个 Document |
|
|
272
|
+
| Excel | `.xlsx` | openpyxl | 每个 sheet 一个 Document |
|
|
273
|
+
|
|
274
|
+
**按后缀自动路由**
|
|
275
|
+
|
|
276
|
+
```python
|
|
277
|
+
from ragkit.readers import read
|
|
278
|
+
docs = read("report.pdf") # 自动选 PDFReader
|
|
279
|
+
docs = read("report.pdf", reader="pdf") # 指定注册名
|
|
280
|
+
docs = read("report.pdf", reader=MyPDFReader()) # 指定实例
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
**PDF 选项**
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
from ragkit.readers import PDFReader
|
|
287
|
+
|
|
288
|
+
reader = PDFReader(
|
|
289
|
+
drop_header_footer=True, # 跨页重复行按页眉页脚去除
|
|
290
|
+
drop_toc=True, # 目录页识别并跳过
|
|
291
|
+
extract_tables=True, # 版面表格转 Markdown
|
|
292
|
+
ocr_fn=lambda i, png: my_ocr(png), # 扫描件 OCR 钩子
|
|
293
|
+
min_text_chars=30, # 少于此字符数视为疑似扫描件
|
|
294
|
+
)
|
|
295
|
+
pipe = RAGPipeline(reader=reader)
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
**Word 解析细节**:按文档流顺序输出「段落 + 表格」,`Heading 1` / `标题 1` / `Title` 样式自动转 `#`,页眉页脚文本会被剔除。
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
### 2. 切分
|
|
303
|
+
|
|
304
|
+
| 策略 | 注册名 | 适用 |
|
|
305
|
+
|---|---|---|
|
|
306
|
+
| 递归字符 | `recursive` | 纯文本、PDF、PPT |
|
|
307
|
+
| 标题层级 | `markdown` | Markdown、Word、HTML、EPUB |
|
|
308
|
+
| 表格 | `table` | CSV、Excel |
|
|
309
|
+
| 问答对 | `qa` | JSON 问答 |
|
|
310
|
+
|
|
311
|
+
```python
|
|
312
|
+
from ragkit import SplitConfig
|
|
313
|
+
|
|
314
|
+
SplitConfig(
|
|
315
|
+
chunk_size=800, # 块长上限
|
|
316
|
+
chunk_overlap=120, # 重叠(必须 < chunk_size)
|
|
317
|
+
min_chunk_size=80, # 过小的块会与相邻块合并
|
|
318
|
+
)
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
**换切分策略**
|
|
322
|
+
|
|
323
|
+
```python
|
|
324
|
+
from ragkit.splitters import splitter_for, RecursiveSplitter
|
|
325
|
+
|
|
326
|
+
pipe.ingest(path, splitter=splitter_for("pdf")) # 按类型
|
|
327
|
+
pipe.ingest(path, splitter=RecursiveSplitter(SplitConfig(chunk_size=500))) # 强制指定
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
**Markdown 结构化切分**保证:
|
|
331
|
+
- 代码块(``` / ~~~)与表格整体保留,绝不从中间切断
|
|
332
|
+
- 块内前缀带完整标题路径:`> 路径:用户指南 / 安装 / 参数配置`
|
|
333
|
+
- 只有超长段落才递归切分
|
|
334
|
+
|
|
335
|
+
**表格切分**:小表整表保留;超过 `whole_table_max_chars` 的大表按「表头 + N 行」复合块拆分,**每块重复表头**。
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
339
|
+
### 3. 嵌入与向量库
|
|
340
|
+
|
|
341
|
+
**接入真实嵌入模型**——继承 `BaseEmbedder`,输出必须 L2 归一化:
|
|
342
|
+
|
|
343
|
+
```python
|
|
344
|
+
from ragkit import BaseEmbedder
|
|
345
|
+
from ragkit.core.embeddings import l2_normalize
|
|
346
|
+
|
|
347
|
+
class MyEmbedder(BaseEmbedder):
|
|
348
|
+
name = "my-embedder"
|
|
349
|
+
dimension = 1024
|
|
350
|
+
|
|
351
|
+
def embed(self, texts):
|
|
352
|
+
vectors = my_model.encode(texts) # 你的模型 / API
|
|
353
|
+
return [l2_normalize(list(map(float, v))) for v in vectors]
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
> 内置 `HashEmbedder` 仅用于测试与演示,区分度差,**不要用于生产**。
|
|
357
|
+
|
|
358
|
+
**本地 Ollama 嵌入(零第三方依赖)**
|
|
359
|
+
|
|
360
|
+
框架内置 `OllamaEmbedder`,走本机 Ollama 的 `/api/embed`,支持批量、自动学习维度、L2 归一化:
|
|
361
|
+
|
|
362
|
+
```bash
|
|
363
|
+
ollama serve
|
|
364
|
+
ollama pull bge-m3 # 1024 维;也可用 nomic-embed-text(768 维)
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
```python
|
|
368
|
+
from ragkit import OllamaEmbedder, RAGPipeline, PipelineConfig
|
|
369
|
+
|
|
370
|
+
embedder = OllamaEmbedder(
|
|
371
|
+
model="bge-m3", # 或 nomic-embed-text
|
|
372
|
+
base_url="http://localhost:11434",
|
|
373
|
+
dimension=None, # 首次调用后自动确定;也可显式填
|
|
374
|
+
timeout=120, batch_size=8,
|
|
375
|
+
)
|
|
376
|
+
pipe = RAGPipeline(PipelineConfig(embed_model_id="ollama/bge-m3"),
|
|
377
|
+
embedder=embedder)
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
模型缺失时提示 `ollama pull <model>`,服务不可用时提示 `ollama serve`。也可以按注册名创建:
|
|
381
|
+
|
|
382
|
+
```python
|
|
383
|
+
from ragkit.core.embeddings import EMBEDDERS
|
|
384
|
+
embedder = EMBEDDERS.create("ollama", model="bge-m3")
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
维度会在首次嵌入后回填到主表 `FileRecord.embed_dimension`,与 `embed_model_id` / `normalization` 一起构成证据链。完整示例见 [`examples/11_ollama.py`](examples/11_ollama.py);与 Weaviate 组合的端到端示例见 [`examples/12_ollama_weaviate.py`](examples/12_ollama_weaviate.py)——先用 embedder 探测维度再建 collection,避免不同模型的维度混用。
|
|
388
|
+
|
|
389
|
+
**三种向量库后端**
|
|
390
|
+
|
|
391
|
+
```python
|
|
392
|
+
from ragkit import RAGPipeline, InMemoryChunkStore
|
|
393
|
+
from ragkit.store import InMemoryVectorStore, QdrantVectorStore, WeaviateVectorStore, create_store
|
|
394
|
+
|
|
395
|
+
# 内存(默认,零依赖)
|
|
396
|
+
pipe = RAGPipeline(store=InMemoryVectorStore())
|
|
397
|
+
|
|
398
|
+
# Qdrant
|
|
399
|
+
pipe = RAGPipeline(store=QdrantVectorStore(
|
|
400
|
+
collection="ragkit_chunks", file_collection="ragkit_files",
|
|
401
|
+
url="http://localhost:6333", dimension=1024, oversample=5))
|
|
402
|
+
|
|
403
|
+
# Weaviate
|
|
404
|
+
pipe = RAGPipeline(store=WeaviateVectorStore(
|
|
405
|
+
class_name="RagChunk", file_class_name="RagFile",
|
|
406
|
+
url="https://xxx.weaviate.cloud", api_key="..."))
|
|
407
|
+
|
|
408
|
+
# 按名创建
|
|
409
|
+
store = create_store("qdrant", collection="rag", url="http://localhost:6333")
|
|
410
|
+
```
|
|
411
|
+
|
|
412
|
+
**后端实现要点**
|
|
413
|
+
|
|
414
|
+
- 每个后端承载**两张表**:chunk 表(带向量、负责检索)+ 文档表(`FileRecord`,负责版本管理)
|
|
415
|
+
- 表字段由 `FileRecord` / `ChunkRecord` 的 dataclass 定义**自动派生**,三方扩展字段会自动成为新列
|
|
416
|
+
- `filter_fn` 是任意 Python 谓词,无法通用下推到各库,采用 **oversample → 本地过滤 → 截断 top_k**,与内存实现行为一致
|
|
417
|
+
- `list_chunks()` 支撑附表审计/回滚:Qdrant 用 `scroll`,Weaviate 用 `fetch_objects`
|
|
418
|
+
|
|
419
|
+
---
|
|
420
|
+
|
|
421
|
+
### 4. 召回
|
|
422
|
+
|
|
423
|
+
```python
|
|
424
|
+
from ragkit import RetrievalConfig
|
|
425
|
+
|
|
426
|
+
RetrievalConfig(
|
|
427
|
+
mode="hybrid", # hybrid(默认)/ vector / keyword
|
|
428
|
+
top_k=10, # 最终返回条数
|
|
429
|
+
candidate_k=50, # 每路先取多少再融合
|
|
430
|
+
rrf_k=60, # RRF 平滑常数
|
|
431
|
+
rrf_weights={"vector": 1.0, "keyword": 0.8},
|
|
432
|
+
enable_prefilter=True # 是否做版本/权限前置过滤
|
|
433
|
+
)
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
**RRF 融合自己实现**(`retrieval/fusion.py`):
|
|
437
|
+
|
|
438
|
+
```
|
|
439
|
+
score(d) = Σᵢ wᵢ / (k + rankᵢ(d))
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
未出现在某路结果中的文档该路贡献 0;也可单独调用:
|
|
443
|
+
|
|
444
|
+
```python
|
|
445
|
+
from ragkit.retrieval import reciprocal_rank_fusion
|
|
446
|
+
|
|
447
|
+
fused = reciprocal_rank_fusion(
|
|
448
|
+
[vector_hits, keyword_hits], k=60,
|
|
449
|
+
weights=[1.0, 0.8], sources=["vector", "keyword"])
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
**关键词召回**是自实现的 BM25(纯 Python,CJK 单字 + 英文词分token),无需外部依赖。
|
|
453
|
+
|
|
454
|
+
---
|
|
455
|
+
|
|
456
|
+
### 5. 重排
|
|
457
|
+
|
|
458
|
+
| 实现 | 注册名 | 依赖 | 说明 |
|
|
459
|
+
|---|---|---|---|
|
|
460
|
+
| 规则重排 | `rule` | 无 | 字面覆盖率 + 标题路径匹配,默认与兜底 |
|
|
461
|
+
| CrossEncoder | `cross_encoder` | sentence-transformers | 语义重排,效果最好 |
|
|
462
|
+
| LLM 重排 | `llm` | 注入 callable | `scorer(query, texts) -> List[float]` |
|
|
463
|
+
|
|
464
|
+
```python
|
|
465
|
+
from ragkit import PipelineConfig, RerankConfig
|
|
466
|
+
from ragkit.rerank import RuleReranker, CrossEncoderReranker, LLMReranker
|
|
467
|
+
|
|
468
|
+
# 默认规则重排(不传 reranker 即为 RuleReranker)
|
|
469
|
+
pipe = RAGPipeline(PipelineConfig(
|
|
470
|
+
rerank=RerankConfig(enabled=True, timeout=2.0, fallback="keep_order", top_n=5)))
|
|
471
|
+
|
|
472
|
+
# CrossEncoder
|
|
473
|
+
pipe = RAGPipeline(reranker=CrossEncoderReranker(
|
|
474
|
+
RerankConfig(timeout=3.0), model_name="cross-encoder/ms-marco-MiniLM-L-6-v2"))
|
|
475
|
+
|
|
476
|
+
# 自定义 LLM 打分
|
|
477
|
+
pipe = RAGPipeline(reranker=LLMReranker(
|
|
478
|
+
RerankConfig(timeout=5.0),
|
|
479
|
+
scorer=lambda q, texts: my_llm_score(q, texts)))
|
|
480
|
+
|
|
481
|
+
# 关闭重排
|
|
482
|
+
pipe = RAGPipeline(PipelineConfig(rerank=RerankConfig(enabled=False)))
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
**超时兜底**:重排超过 `timeout` 或抛异常时,自动按 `fallback` 返回:
|
|
486
|
+
- `keep_order`:保留召回原顺序
|
|
487
|
+
- `keep_score`:保留召回原分数
|
|
488
|
+
|
|
489
|
+
兜底原因记录在 `reranker.last_fallback`,并会触发 `rerank:fallback` 事件(`reason` = `timeout` / `error` / `invalid_scores`)。
|
|
490
|
+
|
|
491
|
+
---
|
|
492
|
+
|
|
493
|
+
### 6. 主表 + 附表(多版本)
|
|
494
|
+
|
|
495
|
+
**主表 `FileRecord`**:`business_id`(业务 id,跨版本稳定)、`version`、`status`、`owner`、`md5`、`file_name`、`created_at`、`updated_at`、`extra`。
|
|
496
|
+
|
|
497
|
+
**证据链字段**(入库时自动填充,也可显式指定):`embed_model_id`(嵌入模型标识)、`embed_dimension`(维度)、`normalization`(归一化方式)、`chunk_strategy`(切分策略)、`chunk_size` / `chunk_overlap`。换模型或改切分参数时,能据此判断该版本要不要重建。`(business_id, version)` 唯一。
|
|
498
|
+
|
|
499
|
+
**`md5` 会自动计算**:文件按字节流式计算、目录按「相对路径 + 各文件 MD5」聚合、`Document` 按文本内容。未显式传入时自动填充,显式传入则以其为准。
|
|
500
|
+
|
|
501
|
+
**内容未变则跳过**(默认开启,由 `PipelineConfig.reingest_unchanged` 控制):
|
|
502
|
+
|
|
503
|
+
- 同一 `(business_id, version)` 且 MD5 相同 → 跳过解析与向量化,`ingest()` 返回空列表,并发出 `ingest:skipped` 事件
|
|
504
|
+
- MD5 不同 → 正常重新解析入库(覆盖该版本)
|
|
505
|
+
- 换版本号(如 `v1` → `v2`)即使内容相同也会入库(版本管理语义)
|
|
506
|
+
- `reingest_unchanged=True`:强制重新解析(例如换了切分参数或嵌入模型)
|
|
507
|
+
|
|
508
|
+
```python
|
|
509
|
+
from ragkit import PipelineConfig
|
|
510
|
+
|
|
511
|
+
# 默认:内容没变就不重复解析
|
|
512
|
+
pipe = RAGPipeline(PipelineConfig())
|
|
513
|
+
pipe.ingest("guide.md", record=FileRecord("guide", "v1")) # 首次,写入 N 块
|
|
514
|
+
pipe.ingest("guide.md", record=FileRecord("guide", "v1")) # 返回 [],命中 MD5 跳过
|
|
515
|
+
|
|
516
|
+
# 强制重新解析
|
|
517
|
+
pipe = RAGPipeline(PipelineConfig(reingest_unchanged=True))
|
|
518
|
+
```
|
|
519
|
+
|
|
520
|
+
**附表 `ChunkRecord`**:`chunk_id`、`business_id`、`version`(外键)、`chunk_index`、`page_no`、`title_path`、`section`、`text`、`acl`、`doc_type`、`language`、`vector`、`metadata`、`extra`。
|
|
521
|
+
|
|
522
|
+
```python
|
|
523
|
+
from ragkit import (RAGPipeline, FileRecord, InMemoryCatalog, InMemoryChunkStore)
|
|
524
|
+
|
|
525
|
+
pipe = RAGPipeline(catalog=InMemoryCatalog(), chunk_store=InMemoryChunkStore())
|
|
526
|
+
|
|
527
|
+
pipe.ingest(doc_v1, record=FileRecord("guide", "v1", status="deprecated",
|
|
528
|
+
owner="alice", md5="aaa"))
|
|
529
|
+
pipe.ingest(doc_v10, record=FileRecord("guide", "v10", status="active",
|
|
530
|
+
owner="alice", md5="bbb"))
|
|
531
|
+
|
|
532
|
+
pipe.retrieve("超时默认值") # 只召回 v10 那组,v1 不召回
|
|
533
|
+
pipe.versions("guide") # ['v1', 'v10']
|
|
534
|
+
rows = pipe.chunks_of("guide", "v1") # List[ChunkRecord],用于审计/回滚
|
|
535
|
+
```
|
|
536
|
+
|
|
537
|
+
**版本择优规则**
|
|
538
|
+
|
|
539
|
+
- 语义化比较,不是字符串比较:`v1.10 > v1.9`、`v3.1.2 > v3.1`
|
|
540
|
+
- 默认在 `status="active"` 里取最高版本;最高版若 `deprecated`,自动退到次高 `active`
|
|
541
|
+
- 不同业务 id 互不影响
|
|
542
|
+
- 与生效时间、ACL 过滤叠加
|
|
543
|
+
|
|
544
|
+
**字段扩展(三方接入必读)**
|
|
545
|
+
|
|
546
|
+
方式一:声明式子类(有类型提示、可生成表列)
|
|
547
|
+
|
|
548
|
+
```python
|
|
549
|
+
from dataclasses import dataclass
|
|
550
|
+
from ragkit import FileRecord
|
|
551
|
+
|
|
552
|
+
@dataclass
|
|
553
|
+
class MyFileRecord(FileRecord):
|
|
554
|
+
department: str = ""
|
|
555
|
+
retention_days: int = 30
|
|
556
|
+
|
|
557
|
+
rec = MyFileRecord("guide", "v1", department="研发", retention_days=90)
|
|
558
|
+
MyFileRecord.columns() # 基类列 + 子类列 + extra
|
|
559
|
+
rec.to_dict() # 扁平字典,含 department / retention_days
|
|
560
|
+
```
|
|
561
|
+
|
|
562
|
+
方式二:`extra` 动态字段
|
|
563
|
+
|
|
564
|
+
```python
|
|
565
|
+
rec = FileRecord("guide", "v1")
|
|
566
|
+
rec.set_extra("tenant", "acme").set_extra("biz_line", "payments")
|
|
567
|
+
rec.get_extra("tenant") # 'acme'
|
|
568
|
+
```
|
|
569
|
+
|
|
570
|
+
两者都会:
|
|
571
|
+
- 出现在 `to_dict()` / `columns()` / 表结构中
|
|
572
|
+
- 自动并入 `to_filter_meta()`,因此**可直接做前置过滤**:
|
|
573
|
+
|
|
574
|
+
```python
|
|
575
|
+
pipe.retrieve("超时", filter_fn=lambda m: m["filter"].get("tenant") == "acme")
|
|
576
|
+
```
|
|
577
|
+
|
|
578
|
+
> 子类新增字段必须给默认值(Python dataclass 继承规则)。`from_dict()` 会把未知键自动归入 `extra`。
|
|
579
|
+
|
|
580
|
+
**主表落真实库**
|
|
581
|
+
|
|
582
|
+
```python
|
|
583
|
+
from ragkit import StoreCatalog, InMemoryChunkStore
|
|
584
|
+
from ragkit.store import WeaviateVectorStore
|
|
585
|
+
|
|
586
|
+
store = WeaviateVectorStore(class_name="RagChunk", file_class_name="RagFile",
|
|
587
|
+
url="...", api_key="...")
|
|
588
|
+
pipe = RAGPipeline(catalog=StoreCatalog(store),
|
|
589
|
+
chunk_store=InMemoryChunkStore(store))
|
|
590
|
+
```
|
|
591
|
+
|
|
592
|
+
**建议的表结构**
|
|
593
|
+
|
|
594
|
+
```sql
|
|
595
|
+
CREATE TABLE files (
|
|
596
|
+
business_id TEXT, version TEXT, status TEXT, owner TEXT,
|
|
597
|
+
md5 TEXT, file_name TEXT, created_at TEXT, updated_at TEXT,
|
|
598
|
+
extra JSON, PRIMARY KEY (business_id, version)
|
|
599
|
+
);
|
|
600
|
+
|
|
601
|
+
CREATE TABLE chunks (
|
|
602
|
+
id TEXT PRIMARY KEY, business_id TEXT, version TEXT,
|
|
603
|
+
chunk_index INT, page_no INT, title_path TEXT, section TEXT,
|
|
604
|
+
text TEXT, acl JSON, doc_type TEXT, language TEXT,
|
|
605
|
+
vector BLOB, metadata JSON, extra JSON,
|
|
606
|
+
FOREIGN KEY (business_id, version) REFERENCES files(business_id, version)
|
|
607
|
+
);
|
|
608
|
+
```
|
|
609
|
+
|
|
610
|
+
建表可直接用 `property_columns(ChunkRecord)` 拿到全部列名:
|
|
611
|
+
|
|
612
|
+
```python
|
|
613
|
+
from ragkit.catalog.scalar import property_columns
|
|
614
|
+
property_columns(ChunkRecord) # ['chunk_id', 'business_id', ..., 'metadata_json', 'extra_json']
|
|
615
|
+
```
|
|
616
|
+
|
|
617
|
+
---
|
|
618
|
+
|
|
619
|
+
### 7. 版本与权限
|
|
620
|
+
|
|
621
|
+
检索前用元数据做**前置过滤**,避免过期/越权内容进入候选集:
|
|
622
|
+
|
|
623
|
+
```python
|
|
624
|
+
from ragkit import PipelineConfig
|
|
625
|
+
|
|
626
|
+
pipe = RAGPipeline(PipelineConfig(
|
|
627
|
+
query_time="2026-06-01T00:00:00Z", # 指定时间点(默认当前)
|
|
628
|
+
user_acls=["dept:eng"], # 用户权限标签
|
|
629
|
+
))
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
**规则**
|
|
633
|
+
|
|
634
|
+
- 生效时间:`effective_from <= now < effective_to`,过期或未生效都不召回
|
|
635
|
+
- 权限:`acl ∩ user_acls ≠ ∅`;`acl` 为空视为公开
|
|
636
|
+
- 叠加版本择优:旧版本同时被排除
|
|
637
|
+
|
|
638
|
+
也可以完全自定义:
|
|
639
|
+
|
|
640
|
+
```python
|
|
641
|
+
pipe.retrieve("q", filter_fn=lambda meta: meta["filter"].get("language") == "zh")
|
|
642
|
+
```
|
|
643
|
+
|
|
644
|
+
---
|
|
645
|
+
|
|
646
|
+
### 8. 回调与进度
|
|
647
|
+
|
|
648
|
+
回调是**只读观察者**:看事件、报错误、追进度,不改变流程结果。
|
|
649
|
+
|
|
650
|
+
**订阅方式**
|
|
651
|
+
|
|
652
|
+
```python
|
|
653
|
+
from ragkit import RAGPipeline, CallbackManager, Event
|
|
654
|
+
|
|
655
|
+
cb = CallbackManager()
|
|
656
|
+
cb.on(Event.ERROR, lambda p: print("失败:", p["stage"], p["error_info"].message))
|
|
657
|
+
cb.on(Event.RETRIEVE_END, lambda p: print("召回", p["count"], "条"))
|
|
658
|
+
|
|
659
|
+
pipe = RAGPipeline(callbacks=cb)
|
|
660
|
+
# 或:callbacks={Event.ERROR: fn} / callbacks=single_callable
|
|
661
|
+
# 或:RAGPipeline(on_error=fn, on_progress=fn)
|
|
662
|
+
```
|
|
663
|
+
|
|
664
|
+
**事件表**
|
|
665
|
+
|
|
666
|
+
| 阶段 | 事件 | 关键字段 |
|
|
667
|
+
|---|---|---|
|
|
668
|
+
| 入库 | `ingest:start` / `ingest:end` | `source` / `count`, `duration_ms`, `md5` |
|
|
669
|
+
| 跳过入库 | `ingest:skipped` | `business_id`, `version`, `md5`, `reason` |
|
|
670
|
+
| 进度 | `ingest:progress` | `stage`, `current`, `total`, `percent`, `eta_ms` |
|
|
671
|
+
| 读取 | `read:end` | `documents`, `count` |
|
|
672
|
+
| 切分 | `split:end` | `chunks`, `count`, `strategy` |
|
|
673
|
+
| 嵌入 | `embed:end` | `count`, `dimension` |
|
|
674
|
+
| 落库 | `store:end` | `count` |
|
|
675
|
+
| 检索 | `retrieve:start` / `retrieve:end` | `query`, `top_k`, `results` |
|
|
676
|
+
| 召回 | `recall:end` | `results`, `count` |
|
|
677
|
+
| 重排 | `rerank:end` | `results`, `fallback` |
|
|
678
|
+
| 重排兜底 | `rerank:fallback` | `reason` |
|
|
679
|
+
| 错误 | `error` / `ingest:error` / `retrieve:error` | `stage`, `error_info` |
|
|
680
|
+
|
|
681
|
+
**实时进度(防"看起来卡死")**
|
|
682
|
+
|
|
683
|
+
大文件入库时,向量化和写库分批处理,每批上报:
|
|
684
|
+
|
|
685
|
+
```python
|
|
686
|
+
pipe = RAGPipeline(
|
|
687
|
+
PipelineConfig(progress_batch_size=32, progress_interval=0.2),
|
|
688
|
+
on_progress=lambda p: print(
|
|
689
|
+
f"{p['stage']} {p['current']}/{p['total']} {p['percent']}% "
|
|
690
|
+
f"ETA {p['eta_ms']}ms"),
|
|
691
|
+
)
|
|
692
|
+
```
|
|
693
|
+
|
|
694
|
+
- `progress_batch_size`:每批块数(越小越细,回调越频繁)
|
|
695
|
+
- `progress_interval`:节流间隔(默认 0.2s),**到 100% 一定强制发一次**
|
|
696
|
+
- 未注册 `on_progress` 时零开销
|
|
697
|
+
|
|
698
|
+
**内置回调**
|
|
699
|
+
|
|
700
|
+
| 类 | 用途 |
|
|
701
|
+
|---|---|
|
|
702
|
+
| `LoggingCallbacks` | 打日志 |
|
|
703
|
+
| `TimingCallbacks` | 阶段耗时汇总 `summary()` |
|
|
704
|
+
| `TraceCollector` | 全链路 trace `dump()` |
|
|
705
|
+
| `ProgressBarCallbacks` | 终端进度条(原地刷新) |
|
|
706
|
+
| `MetricsCallbacks` | 计数/错误/耗时 `snapshot()`,接监控 |
|
|
707
|
+
| `JsonlAuditCallbacks` | 事件写 JSONL 审计文件 |
|
|
708
|
+
| `ErrorAlertCallbacks` | 错误告警(注入 notifier,带冷却防风暴) |
|
|
709
|
+
| `SamplingCallbacks` | 按概率采样降噪 |
|
|
710
|
+
|
|
711
|
+
```python
|
|
712
|
+
from ragkit import (CallbackManager, Event, MetricsCallbacks, TimingCallbacks,
|
|
713
|
+
TraceCollector, ProgressBarCallbacks, JsonlAuditCallbacks,
|
|
714
|
+
ErrorAlertCallbacks)
|
|
715
|
+
|
|
716
|
+
cb = CallbackManager()
|
|
717
|
+
metrics = MetricsCallbacks(); metrics.attach_to(cb) # 全量指标
|
|
718
|
+
timing = TimingCallbacks()
|
|
719
|
+
cb.on(Event.INGEST_END, timing); cb.on(Event.RETRIEVE_END, timing)
|
|
720
|
+
cb.on(Event.PROGRESS, ProgressBarCallbacks()) # 进度条
|
|
721
|
+
JsonlAuditCallbacks("audit.jsonl").attach_to(cb) # 审计落盘
|
|
722
|
+
cb.on(Event.ERROR, ErrorAlertCallbacks(lambda info: send_alert(info.to_dict())))
|
|
723
|
+
|
|
724
|
+
pipe = RAGPipeline(callbacks=cb)
|
|
725
|
+
metrics.snapshot() # {"events": {...}, "errors": {...}, "durations": {...}}
|
|
726
|
+
timing.summary() # {"retrieve:end": {"count": 3, "avg_ms": 12.4, ...}}
|
|
727
|
+
```
|
|
728
|
+
|
|
729
|
+
> 回调自身抛错不会影响主流程,记录在 `cb.callback_errors`。
|
|
730
|
+
|
|
731
|
+
---
|
|
732
|
+
|
|
733
|
+
### 9. 中间件
|
|
734
|
+
|
|
735
|
+
中间件是**拦截器**:能改参数、改结果、短路、重试、熔断。
|
|
736
|
+
|
|
737
|
+
```python
|
|
738
|
+
from ragkit import (RAGPipeline, BaseMiddleware, MiddlewareChain,
|
|
739
|
+
GuardMiddleware, QueryNormalizeMiddleware, TopKClampMiddleware,
|
|
740
|
+
CacheMiddleware, DedupMiddleware, RetryMiddleware,
|
|
741
|
+
ResultFilterMiddleware, CircuitBreakerMiddleware, AuditMiddleware)
|
|
742
|
+
|
|
743
|
+
class RewriteQuery(BaseMiddleware):
|
|
744
|
+
stage_filter = {"retrieve"} # 只在检索阶段生效
|
|
745
|
+
def before(self, stage, ctx):
|
|
746
|
+
ctx["query"] = ctx["query"] + " 配置项" # 改查询
|
|
747
|
+
ctx["top_k"] = 5 # 调召回数
|
|
748
|
+
|
|
749
|
+
class DropLowScore(BaseMiddleware):
|
|
750
|
+
stage_filter = {"retrieve"}
|
|
751
|
+
def after(self, stage, ctx, result):
|
|
752
|
+
return [r for r in result if r.score > 0.3] # 改结果
|
|
753
|
+
|
|
754
|
+
pipe = RAGPipeline(middleware=[
|
|
755
|
+
GuardMiddleware(), # 空查询短路
|
|
756
|
+
QueryNormalizeMiddleware(), # 全角转半角、折叠空白
|
|
757
|
+
TopKClampMiddleware(max_k=20), # 防 top_k 过大(要在缓存之前)
|
|
758
|
+
CacheMiddleware(), # 相同 query 走缓存
|
|
759
|
+
RewriteQuery(),
|
|
760
|
+
DedupMiddleware(), # chunk_id 去重
|
|
761
|
+
DropLowScore(),
|
|
762
|
+
CircuitBreakerMiddleware(threshold=5, cooldown=30),
|
|
763
|
+
RetryMiddleware(times=2, stages={"ingest"}),
|
|
764
|
+
AuditMiddleware(sink=my_audit_sink),
|
|
765
|
+
])
|
|
766
|
+
```
|
|
767
|
+
|
|
768
|
+
**三个钩子**
|
|
769
|
+
|
|
770
|
+
| 钩子 | 时机 | 能力 |
|
|
771
|
+
|---|---|---|
|
|
772
|
+
| `before(stage, ctx)` | 阶段前 | 改 `ctx`;返回 `False` 短路(结果取 `ctx["short_circuit"]`) |
|
|
773
|
+
| `after(stage, ctx, result)` | 阶段后 | 返回值替换结果 |
|
|
774
|
+
| `on_error(stage, ctx, exc, retry)` | 阶段抛错 | 返回非 `None` 降级;`RETRY` 或 `retry()` 重试 |
|
|
775
|
+
|
|
776
|
+
**ctx 可改字段**
|
|
777
|
+
|
|
778
|
+
- `ingest`:`source` / `record` / `splitter` / `chunks`
|
|
779
|
+
- `retrieve`:`query` / `top_k` / `filter_fn` / `results`
|
|
780
|
+
|
|
781
|
+
**内置中间件**
|
|
782
|
+
|
|
783
|
+
| 类 | 用途 |
|
|
784
|
+
|---|---|
|
|
785
|
+
| `LoggingMiddleware` | 阶段耗时日志 |
|
|
786
|
+
| `RetryMiddleware` | 失败自动重试 |
|
|
787
|
+
| `CacheMiddleware` | query→结果缓存 |
|
|
788
|
+
| `GuardMiddleware` | 空查询短路 |
|
|
789
|
+
| `ResultFilterMiddleware` | 按谓词裁剪结果 |
|
|
790
|
+
| `DedupMiddleware` | 按 chunk_id 去重 |
|
|
791
|
+
| `TopKClampMiddleware` | 钳制 top_k 上下界 |
|
|
792
|
+
| `QueryNormalizeMiddleware` | NFKC 全角转半角、折叠空白、去标点 |
|
|
793
|
+
| `CircuitBreakerMiddleware` | 连续失败熔断 + 冷却半开 |
|
|
794
|
+
| `AuditMiddleware` | 审计记录(sink 或 logger) |
|
|
795
|
+
|
|
796
|
+
**顺序很关键**:`before` 正序、`after` 逆序(洋葱模型)。
|
|
797
|
+
|
|
798
|
+
缓存类中间件尤其要注意:它的 key 依赖 `query` 和 `top_k`,所以**所有会修改这两个字段的中间件都必须排在缓存之前**。例如:
|
|
799
|
+
|
|
800
|
+
```python
|
|
801
|
+
# 正确
|
|
802
|
+
middleware=[QueryNormalizeMiddleware(), TopKClampMiddleware(max_k=20), CacheMiddleware()]
|
|
803
|
+
# 错误:TopKClamp 在缓存之后,第一次缓存 key 用 100、第二次用 20,永远命中不了
|
|
804
|
+
middleware=[QueryNormalizeMiddleware(), CacheMiddleware(), TopKClampMiddleware(max_k=20)]
|
|
805
|
+
```
|
|
806
|
+
|
|
807
|
+
中间件自身异常会包成 `MiddlewareError`,与业务异常区分。
|
|
808
|
+
|
|
809
|
+
**callback vs middleware**:只需埋点/进度/错误上报 → callback;需要改参数、拦结果、短路、重试 → middleware。
|
|
810
|
+
|
|
811
|
+
---
|
|
812
|
+
|
|
813
|
+
## 扩展指南
|
|
814
|
+
|
|
815
|
+
所有组件都面向基类,两种扩展方式:**注册名**(全局按名引用)或**构造注入**(局部替换)。
|
|
816
|
+
|
|
817
|
+
### 新增读取器
|
|
818
|
+
|
|
819
|
+
```python
|
|
820
|
+
from ragkit.readers.base import BaseReader, READERS
|
|
821
|
+
from ragkit.core.schema import Document
|
|
822
|
+
|
|
823
|
+
@READERS.register("my_format")
|
|
824
|
+
class MyFormatReader(BaseReader):
|
|
825
|
+
name = "my_format"
|
|
826
|
+
|
|
827
|
+
def read(self, source, **kwargs):
|
|
828
|
+
text = parse_my_format(source)
|
|
829
|
+
return [Document(text=text)]
|
|
830
|
+
```
|
|
831
|
+
|
|
832
|
+
按后缀自动路由需在 `readers/__init__.py` 的 `_SUFFIX_MAP` 加一行。
|
|
833
|
+
|
|
834
|
+
### 新增切分器
|
|
835
|
+
|
|
836
|
+
```python
|
|
837
|
+
from ragkit.splitters.base import BaseSplitter, SPLITTERS
|
|
838
|
+
|
|
839
|
+
@SPLITTERS.register("sliding")
|
|
840
|
+
class SlidingSplitter(BaseSplitter):
|
|
841
|
+
name = "sliding"
|
|
842
|
+
|
|
843
|
+
def split(self, document, **kwargs):
|
|
844
|
+
# _sentences 按句/段边界切;_iter_chunks 施加块长/重叠与元数据
|
|
845
|
+
return self._iter_chunks(self._sentences(document.text), document.metadata)
|
|
846
|
+
```
|
|
847
|
+
|
|
848
|
+
### 新增嵌入 / 向量库 / 召回 / 重排
|
|
849
|
+
|
|
850
|
+
```python
|
|
851
|
+
from ragkit import RAGPipeline, BaseEmbedder
|
|
852
|
+
from ragkit.store.base import BaseVectorStore
|
|
853
|
+
from ragkit.retrieval.retrievers import BaseRetriever
|
|
854
|
+
from ragkit.rerank.base import BaseReranker
|
|
855
|
+
|
|
856
|
+
class MyVectorStore(BaseVectorStore): # 接 Milvus / pgvector / 自己实现
|
|
857
|
+
def add(self, chunks, vectors): ...
|
|
858
|
+
def search(self, query_vec, top_k, filter_fn=None): ...
|
|
859
|
+
def clear(self): ...
|
|
860
|
+
def __len__(self): ...
|
|
861
|
+
|
|
862
|
+
class MyReranker(BaseReranker): # 自动获得超时兜底
|
|
863
|
+
def score(self, query, chunks):
|
|
864
|
+
return [my_score(query, c.chunk.text) for c in chunks]
|
|
865
|
+
|
|
866
|
+
pipe = RAGPipeline(
|
|
867
|
+
embedder=MyEmbedder(), # 输出须 L2 归一化
|
|
868
|
+
store=MyVectorStore(),
|
|
869
|
+
retriever=MyRetriever(),
|
|
870
|
+
reranker=MyReranker(),
|
|
871
|
+
)
|
|
872
|
+
```
|
|
873
|
+
|
|
874
|
+
### 自定义表记录
|
|
875
|
+
|
|
876
|
+
```python
|
|
877
|
+
from dataclasses import dataclass
|
|
878
|
+
from ragkit import FileRecord, ChunkRecord
|
|
879
|
+
|
|
880
|
+
@dataclass
|
|
881
|
+
class TenantFile(FileRecord):
|
|
882
|
+
tenant: str = ""
|
|
883
|
+
|
|
884
|
+
@dataclass
|
|
885
|
+
class TenantChunk(ChunkRecord):
|
|
886
|
+
tenant: str = ""
|
|
887
|
+
```
|
|
888
|
+
|
|
889
|
+
自定义 `ChunkRecord` 需同时覆写 `BaseChunkStore` 的写入逻辑,或直接在 `extra` 里挂字段(无需改后端)。
|
|
890
|
+
|
|
891
|
+
---
|
|
892
|
+
|
|
893
|
+
## 配置参考
|
|
894
|
+
|
|
895
|
+
**SplitConfig**
|
|
896
|
+
|
|
897
|
+
| 字段 | 默认 | 说明 |
|
|
898
|
+
|---|---|---|
|
|
899
|
+
| `chunk_size` | 800 | 块长上限 |
|
|
900
|
+
| `chunk_overlap` | 120 | 重叠,须 `< chunk_size` |
|
|
901
|
+
| `min_chunk_size` | 80 | 小于此值的块与相邻合并 |
|
|
902
|
+
|
|
903
|
+
**RetrievalConfig**
|
|
904
|
+
|
|
905
|
+
| 字段 | 默认 | 说明 |
|
|
906
|
+
|---|---|---|
|
|
907
|
+
| `mode` | `hybrid` | `hybrid` / `vector` / `keyword` |
|
|
908
|
+
| `top_k` | 10 | 最终返回数 |
|
|
909
|
+
| `candidate_k` | 50 | 每路候选数 |
|
|
910
|
+
| `rrf_k` | 60 | RRF 平滑常数 |
|
|
911
|
+
| `rrf_weights` | None | 各路权重 |
|
|
912
|
+
| `enable_prefilter` | True | 版本/权限前置过滤 |
|
|
913
|
+
|
|
914
|
+
**RerankConfig**
|
|
915
|
+
|
|
916
|
+
| 字段 | 默认 | 说明 |
|
|
917
|
+
|---|---|---|
|
|
918
|
+
| `enabled` | True | 是否重排 |
|
|
919
|
+
| `timeout` | 2.0 | 超时秒数 |
|
|
920
|
+
| `fallback` | `keep_order` | `keep_order` / `keep_score` |
|
|
921
|
+
| `top_n` | None | 重排后截断 |
|
|
922
|
+
|
|
923
|
+
**PipelineConfig**
|
|
924
|
+
|
|
925
|
+
| 字段 | 默认 | 说明 |
|
|
926
|
+
|---|---|---|
|
|
927
|
+
| `split` / `retrieval` / `rerank` | 见上 | 子配置 |
|
|
928
|
+
| `embed_model_id` | `hash-embedder` | 写入证据链 |
|
|
929
|
+
| `reingest_unchanged` | False | MD5 相同是否仍重新解析入库 |
|
|
930
|
+
| `progress_batch_size` | 32 | 分批大小 |
|
|
931
|
+
| `progress_interval` | 0.2 | 进度节流(秒) |
|
|
932
|
+
| `query_time` | None | 时间点过滤(ISO) |
|
|
933
|
+
| `user_acls` | `[]` | 权限标签 |
|
|
934
|
+
|
|
935
|
+
---
|
|
936
|
+
|
|
937
|
+
## 目录结构
|
|
938
|
+
|
|
939
|
+
```
|
|
940
|
+
ragkit/
|
|
941
|
+
├── core/ 数据模型 / 配置 / 嵌入基类 / 版本权限过滤
|
|
942
|
+
├── readers/ 读取器(base + text/md/json/csv/html/epub/rtf/pdf/office)+ markdown_parser
|
|
943
|
+
├── splitters/ 切分器(base + recursive/heading/table/qa)
|
|
944
|
+
├── store/ 向量库(base + memory/qdrant/weaviate)
|
|
945
|
+
├── catalog/ 主表+附表(models / base / chunk_store / scalar / store_catalog / version)
|
|
946
|
+
├── retrieval/ 召回(base + vector/keyword + RRF 融合 + hybrid 编排)
|
|
947
|
+
├── rerank/ 重排(base + rule/cross_encoder/llm + 超时兜底)
|
|
948
|
+
├── callbacks/ 事件 / 进度 / 错误 / 8 个内置回调
|
|
949
|
+
├── middleware/ 10 个内置中间件
|
|
950
|
+
├── utils/ 注册表 + 超时兜底
|
|
951
|
+
└── pipeline.py 端到端编排
|
|
952
|
+
```
|
|
953
|
+
|
|
954
|
+
---
|
|
955
|
+
|
|
956
|
+
## 测试
|
|
957
|
+
|
|
958
|
+
纯标准库即可运行,无需外部服务(Qdrant/Weaviate 用假 client 验证映射逻辑)。
|
|
959
|
+
|
|
960
|
+
```bash
|
|
961
|
+
python tests/test_smoke.py # 端到端冒烟
|
|
962
|
+
python tests/test_catalog.py # 主表+附表、版本择优
|
|
963
|
+
python tests/test_stores.py # Qdrant/Weaviate 字段映射与两表
|
|
964
|
+
python tests/test_extend.py # 记录字段扩展
|
|
965
|
+
python tests/test_readers.py # 格式解析
|
|
966
|
+
python tests/test_markdown.py # Markdown 结构化解析
|
|
967
|
+
python tests/test_callbacks.py # 回调机制
|
|
968
|
+
python tests/test_progress.py # 实时进度
|
|
969
|
+
python tests/test_middleware.py # 中间件
|
|
970
|
+
python tests/test_builtins.py # 内置回调/中间件
|
|
971
|
+
python tests/test_ollama.py # Ollama 嵌入器(假 HTTP 服务,无需真实 Ollama)
|
|
972
|
+
python tests/test_md5.py # MD5 指纹与内容未变跳过
|
|
973
|
+
python tests/test_usage.py # README 示例可执行性
|
|
974
|
+
```
|
|
975
|
+
|
|
976
|
+
当前共 **361 项断言**,全部通过。
|
|
977
|
+
|
|
978
|
+
示例(按场景递进,全部可直接运行,见 [`examples/README.md`](examples/README.md)):
|
|
979
|
+
|
|
980
|
+
```bash
|
|
981
|
+
python examples/01_quickstart.py # 最小闭环
|
|
982
|
+
python examples/02_multi_format.py # Markdown/CSV/JSON 混合入库
|
|
983
|
+
python examples/03_multi_version.py # 主表+附表多版本、回滚
|
|
984
|
+
python examples/04_recall_compare.py # vector/keyword/hybrid + 重排 A/B
|
|
985
|
+
python examples/05_observability.py # 进度条、指标、trace、审计
|
|
986
|
+
python examples/06_middleware.py # 短路、缓存、钳制、去重、熔断
|
|
987
|
+
python examples/07_custom_extension.py # 自定义各层抽象
|
|
988
|
+
python examples/09_weaviate.py # Weaviate 真实后端(需 Docker)
|
|
989
|
+
python examples/10_weaviate_extended.py # file/chunk 扩展字段 + 证据链
|
|
990
|
+
python examples/11_ollama.py # 本地 Ollama 嵌入模型(需 ollama serve)
|
|
991
|
+
python examples/12_ollama_weaviate.py # Ollama 嵌入 + Weaviate 真实向量库
|
|
992
|
+
```
|
|
993
|
+
|
|
994
|
+
### 用 Docker 跑 Weaviate
|
|
995
|
+
|
|
996
|
+
`weaviate-client 4.24` 要求**服务端 ≥ 1.27**(1.25 及以下会直接拒绝连接)。
|
|
997
|
+
|
|
998
|
+
```bash
|
|
999
|
+
docker run -d --name weaviate-ragkit \
|
|
1000
|
+
-p 8080:8080 -p 50051:50051 \
|
|
1001
|
+
-e AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true \
|
|
1002
|
+
-e PERSISTENCE_DATA_PATH=/var/lib/weaviate \
|
|
1003
|
+
-e DEFAULT_VECTORIZER_MODULE=none \
|
|
1004
|
+
cr.weaviate.io/semitechnologies/weaviate:latest
|
|
1005
|
+
|
|
1006
|
+
# 等待就绪(返回 200 即可)
|
|
1007
|
+
curl -s -o /dev/null -w '%{http_code}\n' http://localhost:8080/v1/.well-known/ready
|
|
1008
|
+
```
|
|
1009
|
+
|
|
1010
|
+
```python
|
|
1011
|
+
import weaviate
|
|
1012
|
+
from ragkit import WeaviateVectorStore, StoreCatalog, InMemoryChunkStore
|
|
1013
|
+
|
|
1014
|
+
client = weaviate.connect_to_local(host="localhost", port=8080, grpc_port=50051)
|
|
1015
|
+
store = WeaviateVectorStore(class_name="RagChunk", file_class_name="RagFile",
|
|
1016
|
+
client=client, dimension=1024)
|
|
1017
|
+
pipe = RAGPipeline(catalog=StoreCatalog(store),
|
|
1018
|
+
chunk_store=InMemoryChunkStore(store))
|
|
1019
|
+
# ... 使用完毕后
|
|
1020
|
+
client.close()
|
|
1021
|
+
```
|
|
1022
|
+
|
|
1023
|
+
清理:`docker stop weaviate-ragkit && docker rm weaviate-ragkit`
|
|
1024
|
+
|
|
1025
|
+
### 可视化管理端(Weaviate Manager)
|
|
1026
|
+
|
|
1027
|
+
官方镜像只有数据库本体。可视化查看用社区开源的 **Weaviate Manager**(中文界面,可浏览 collection / 对象 / 执行 GraphQL 与向量检索)。
|
|
1028
|
+
|
|
1029
|
+
```bash
|
|
1030
|
+
# 方式一:单独拉起管理端(连已有 Weaviate)
|
|
1031
|
+
docker run -d --name weaviate-manager -p 3000:80 \
|
|
1032
|
+
ghcr.io/aicode-world/weaviate-manager:latest
|
|
1033
|
+
|
|
1034
|
+
# 打开 http://localhost:3000,连接地址填 http://localhost:8080
|
|
1035
|
+
```
|
|
1036
|
+
|
|
1037
|
+
```bash
|
|
1038
|
+
# 方式二:一键起数据库 + 管理端(本项目自带 docker-compose.yml)
|
|
1039
|
+
docker compose up -d
|
|
1040
|
+
# Weaviate http://localhost:8080
|
|
1041
|
+
# 管理端 http://localhost:3000
|
|
1042
|
+
```
|
|
1043
|
+
|
|
1044
|
+
⚠️ **必须给 Weaviate 开 CORS**,否则管理端页面能打开但连不上:
|
|
1045
|
+
|
|
1046
|
+
```bash
|
|
1047
|
+
docker run -d ... -e CORS_ALLOWED_ORIGINS='*' cr.weaviate.io/semitechnologies/weaviate:latest
|
|
1048
|
+
```
|
|
1049
|
+
|
|
1050
|
+
验证 CORS 是否生效:
|
|
1051
|
+
|
|
1052
|
+
```bash
|
|
1053
|
+
curl -s -I http://localhost:8080/v1/schema -H 'Origin: http://localhost:3000' \
|
|
1054
|
+
| grep -i access-control-allow-origin
|
|
1055
|
+
# 应输出 Access-Control-Allow-Origin: *
|
|
1056
|
+
```
|
|
1057
|
+
|
|
1058
|
+
其他可选项:`Weavit UI`(桌面客户端)、`LibreDB Studio`(只读浏览器)。
|
|
1059
|
+
|
|
1060
|
+
---
|
|
1061
|
+
|
|
1062
|
+
## 注意事项
|
|
1063
|
+
|
|
1064
|
+
1. **不要用 `HashEmbedder` 上生产**——它是确定性哈希,只保证接口可用,区分度很差。
|
|
1065
|
+
2. **向量必须在入库前 L2 归一化**,否则余弦相似度会失真。
|
|
1066
|
+
3. **`chunk_overlap` 必须小于 `chunk_size`**,否则构造配置时直接报错。
|
|
1067
|
+
4. **关键词索引不要手写分批重建**——`KeywordRetriever.add()` 只累积语料,检索前统一重建;多次 `ingest` 的语料会自动累加,所以混合召回在有主表/附表时也能正常工作。
|
|
1068
|
+
5. **重排兜底是静默的**,如果发现重排没生效,去看 `rerank:fallback` 事件的 `reason`。
|
|
1069
|
+
6. **中间件顺序有意义**:所有会修改 `query` / `top_k` 的中间件必须排在缓存之前,否则缓存永远命中不了。
|
|
1070
|
+
7. **`ingest` 支持目录**,会递归读取已知后缀并跳过隐藏文件/未知格式。
|
|
1071
|
+
8. **多版本必须给 `business_id` + `version`**,否则不登记主表,也就没有版本择优。
|
|
1072
|
+
9. **PDF 扫描件需要自己提供 OCR**,库只负责调用钩子,不捆绑 OCR 引擎。
|
|
1073
|
+
10. **无需安装即可使用核心功能**;`pip install -e .` 或 `pip install -e ".[all]"` 安装可选依赖。
|