agg-search 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agg_search-0.1.0/LICENSE +21 -0
- agg_search-0.1.0/PKG-INFO +147 -0
- agg_search-0.1.0/README.md +130 -0
- agg_search-0.1.0/pyproject.toml +34 -0
- agg_search-0.1.0/setup.cfg +4 -0
- agg_search-0.1.0/src/agg_search/__init__.py +0 -0
- agg_search-0.1.0/src/agg_search/eval.py +165 -0
- agg_search-0.1.0/src/agg_search/fetch.py +226 -0
- agg_search-0.1.0/src/agg_search/fetch_container.py +150 -0
- agg_search-0.1.0/src/agg_search/mcp.py +171 -0
- agg_search-0.1.0/src/agg_search/search.py +934 -0
- agg_search-0.1.0/src/agg_search.egg-info/PKG-INFO +147 -0
- agg_search-0.1.0/src/agg_search.egg-info/SOURCES.txt +16 -0
- agg_search-0.1.0/src/agg_search.egg-info/dependency_links.txt +1 -0
- agg_search-0.1.0/src/agg_search.egg-info/entry_points.txt +5 -0
- agg_search-0.1.0/src/agg_search.egg-info/top_level.txt +1 -0
- agg_search-0.1.0/tests/test_mcp.py +52 -0
- agg_search-0.1.0/tests/test_offline.py +129 -0
agg_search-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TsukimioRinka
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agg-search
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Multi-engine web search with RRF fusion, Jina reranking, query expansion and a tiered fetcher — CLI + MCP server, stdlib only.
|
|
5
|
+
Author: TsukimioRinka
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/TsukimioRinka/agg-search
|
|
8
|
+
Keywords: search,mcp,rrf,rerank,agents,rag
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# agg-search
|
|
19
|
+
|
|
20
|
+
[](https://github.com/TsukimioRinka/agg-search/actions/workflows/tests.yml)
|
|
21
|
+
[](https://github.com/TsukimioRinka/agg-search/releases)
|
|
22
|
+
[](LICENSE)
|
|
23
|
+
[](pyproject.toml)
|
|
24
|
+
|
|
25
|
+
多引擎并行召回 + RRF 融合 + Jina 精排的搜索工具链:**CLI + MCP server + 分层抓取**,宿主机零依赖(纯 Python 标准库)。
|
|
26
|
+
|
|
27
|
+
A multi-engine search stack for agents: parallel recall across eight engines, RRF fusion, listwise reranking, optional query expansion and cited answers, plus a tiered fetcher. Ships as a CLI and an MCP server. Standard library only on the host.
|
|
28
|
+
|
|
29
|
+

|
|
30
|
+
|
|
31
|
+
*真实一次运行的输出:7 个引擎返回结果,DeepSeek 因余额不足进入冷却、被跳过;空结果不阻塞其余引擎。*
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## 为什么又一个搜索聚合器
|
|
36
|
+
|
|
37
|
+
同类项目不少,但多数是「多引擎 + 简单合并」或「零 key 依赖单一索引」。这个项目的差异在四件事:
|
|
38
|
+
|
|
39
|
+
1. **DeepSeek 搜索/抓取是一等公民** —— 官方 `/v1/tools/search` 自带 `rerank_score`,`/v1/tools/fetch` 服务端取页直接给 Markdown,省掉容器。
|
|
40
|
+
2. **抓取是阶梯而不是一个函数** —— DeepSeek Fetch → 静态 → 动态 → 反爬,逐级升级,跑在隔离容器里,宿主机不装浏览器。
|
|
41
|
+
3. **排序是两阶段** —— RRF 粗排前 15,再交给 Jina Reranker v3.5 listwise 精排;模型不可用链式回退 v3 → v2,全失败则保留 RRF 顺序,绝不阻塞。
|
|
42
|
+
4. **韧性层是显式的** —— 5 分钟磁盘缓存、按错误类型分级的冷却(额度 1h / 反爬 10min / 超时 90s)、引擎健康上报,缺 key 的引擎提前剔除不占线程。
|
|
43
|
+
|
|
44
|
+
## 架构
|
|
45
|
+
|
|
46
|
+
```mermaid
|
|
47
|
+
flowchart LR
|
|
48
|
+
Q[query] --> X[query expansion<br/>local ollama]
|
|
49
|
+
X --> A[exa]
|
|
50
|
+
X --> B[brave]
|
|
51
|
+
X --> C[serper]
|
|
52
|
+
X --> D[deepseek]
|
|
53
|
+
X --> E[firecrawl]
|
|
54
|
+
X --> F[anysearch]
|
|
55
|
+
X --> G[tavily]
|
|
56
|
+
X --> H[searxng]
|
|
57
|
+
A & B & C & D & E & F & G & H --> R[RRF fusion<br/>top 15]
|
|
58
|
+
R --> J[jina rerank v3.5<br/>fallback v3 → v2]
|
|
59
|
+
J --> O[top N results<br/>+ provenance + health]
|
|
60
|
+
O --> AN[optional cited answer]
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## 安装
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
git clone https://github.com/TsukimioRinka/agg-search
|
|
67
|
+
cd agg-search
|
|
68
|
+
python3 -m pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
只想要脚本、不想装包也行:`PYTHONPATH=src python3 -m agg_search.search "query"`。
|
|
72
|
+
|
|
73
|
+
## 使用
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# 搜索
|
|
77
|
+
agg-search "rust async runtime comparison" --top-n 10 --recency month
|
|
78
|
+
|
|
79
|
+
# 机器可读输出
|
|
80
|
+
agg-search "MCP server examples" --json
|
|
81
|
+
|
|
82
|
+
# RAG-Fusion 查询扩展(需要本地 ollama)
|
|
83
|
+
agg-search "vector database tradeoffs" --expand 3
|
|
84
|
+
|
|
85
|
+
# 带引用的答案段(本地 ollama;--answer deepseek 走 API)
|
|
86
|
+
agg-search "what changed in Python 3.13" --answer
|
|
87
|
+
|
|
88
|
+
# 快速模式:单池扇出,最早两个有结果的引擎就返回
|
|
89
|
+
agg-search "breaking news" --fast
|
|
90
|
+
|
|
91
|
+
# 抓取正文:DeepSeek Fetch 优先,失败落容器阶梯
|
|
92
|
+
crawl-fetch --url https://example.com/post --tier auto --out md
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
密钥只从环境变量读(也兼容 `~/.config/agent-env`、`~/.config/ai-keys.env` 的 `KEY=` 行)。缺失的引擎会被跳过,不会让整次搜索失败。完整变量表见 [docs/environment.md](docs/environment.md)。
|
|
96
|
+
|
|
97
|
+
## MCP server
|
|
98
|
+
|
|
99
|
+
```json
|
|
100
|
+
{
|
|
101
|
+
"mcpServers": {
|
|
102
|
+
"agg-search": {
|
|
103
|
+
"command": "agg-search-mcp",
|
|
104
|
+
"env": { "BRAVE_API_KEY": "...", "EXA_API_KEY": "..." }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
暴露两个工具:
|
|
111
|
+
|
|
112
|
+
| 工具 | 参数 | 说明 |
|
|
113
|
+
|---|---|---|
|
|
114
|
+
| `search` | `query`(必填)、`top_n`、`recency`、`engines`、`expand`、`answer`、`fast`、`no_jina` | 并行召回 + RRF + 精排,返回 JSON |
|
|
115
|
+
| `fetch` | `urls`(必填)、`tier`、`out`、`text_limit` | 分层抓取正文 |
|
|
116
|
+
|
|
117
|
+
服务用换行分隔的 JSON-RPC 2.0,`initialize` / `tools/list` / `tools/call` / `ping` 都已实现,可直接挂到 MCP 客户端。
|
|
118
|
+
|
|
119
|
+
## 同类项目对比
|
|
120
|
+
|
|
121
|
+
只列能核实的能力,语言一列仅在仓库自述里明确时填写:
|
|
122
|
+
|
|
123
|
+
| 项目 | 栈 | 它强在哪 | 与本项目的差别 |
|
|
124
|
+
|---|---|---|---|
|
|
125
|
+
| [search-cli](https://github.com/paperfoot/search-cli) | Rust | 13 个 provider,单一二进制 | 本项目免编译;多容器抓取阶梯与 DeepSeek Fetch 是一等路径 |
|
|
126
|
+
| [web-search-mcp](https://github.com/charley-forey/web-search-mcp) | TypeScript | SSRF 安全抓取、注入防御(`<untrusted_content>` 包裹 + 隐藏字符剥离)、PDF/RSS/YouTube 抽取、本地语义重排、Recall@5 / MRR@10 评测、MCP resources/prompts | 它的抓取安全与内容抽取比本项目完整;本项目的差异在 8 引擎并行 + RRF→Jina listwise 两阶段排序、DeepSeek Search/Fetch、RAG-Fusion 扩展与容器化的静态→动态→反爬阶梯 |
|
|
127
|
+
| [argo](https://github.com/taxueseek/argo) | — | 300+ 来源、18 语言、MCP/CLI | 本项目只做通用网页检索,覆盖面窄;差异在排序链路与成本控制更细 |
|
|
128
|
+
| [websearch-mcpserver](https://github.com/daidaiJ/websearch-mcpserver) | Go | 零 key 可跑、多引擎 + 学术源 | 本项目按 key 选择性启用引擎,不承诺零 key |
|
|
129
|
+
| [free-search-api](https://github.com/Ethos6/free-search-api) | — | 自托管、无 key、Tavily 兼容端点 | 本项目是客户端编排层,不提供兼容 API |
|
|
130
|
+
|
|
131
|
+
## 测试
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
离线测试只覆盖纯函数(域名归并、去重、RRF、冷却分类、缓存键),不发起网络请求、不消耗额度。
|
|
138
|
+
|
|
139
|
+
## 安全与隐私
|
|
140
|
+
|
|
141
|
+
- 仓库内不含任何密钥、账号、内网地址;`.env.example` 只有空占位。
|
|
142
|
+
- 密钥只在进程环境里读取,不写日志、不落盘。
|
|
143
|
+
- 抓取在容器内执行,宿主机不留浏览器与爬虫依赖。
|
|
144
|
+
|
|
145
|
+
## License
|
|
146
|
+
|
|
147
|
+
MIT
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# agg-search
|
|
2
|
+
|
|
3
|
+
[](https://github.com/TsukimioRinka/agg-search/actions/workflows/tests.yml)
|
|
4
|
+
[](https://github.com/TsukimioRinka/agg-search/releases)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](pyproject.toml)
|
|
7
|
+
|
|
8
|
+
多引擎并行召回 + RRF 融合 + Jina 精排的搜索工具链:**CLI + MCP server + 分层抓取**,宿主机零依赖(纯 Python 标准库)。
|
|
9
|
+
|
|
10
|
+
A multi-engine search stack for agents: parallel recall across eight engines, RRF fusion, listwise reranking, optional query expansion and cited answers, plus a tiered fetcher. Ships as a CLI and an MCP server. Standard library only on the host.
|
|
11
|
+
|
|
12
|
+

|
|
13
|
+
|
|
14
|
+
*真实一次运行的输出:7 个引擎返回结果,DeepSeek 因余额不足进入冷却、被跳过;空结果不阻塞其余引擎。*
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## 为什么又一个搜索聚合器
|
|
19
|
+
|
|
20
|
+
同类项目不少,但多数是「多引擎 + 简单合并」或「零 key 依赖单一索引」。这个项目的差异在四件事:
|
|
21
|
+
|
|
22
|
+
1. **DeepSeek 搜索/抓取是一等公民** —— 官方 `/v1/tools/search` 自带 `rerank_score`,`/v1/tools/fetch` 服务端取页直接给 Markdown,省掉容器。
|
|
23
|
+
2. **抓取是阶梯而不是一个函数** —— DeepSeek Fetch → 静态 → 动态 → 反爬,逐级升级,跑在隔离容器里,宿主机不装浏览器。
|
|
24
|
+
3. **排序是两阶段** —— RRF 粗排前 15,再交给 Jina Reranker v3.5 listwise 精排;模型不可用链式回退 v3 → v2,全失败则保留 RRF 顺序,绝不阻塞。
|
|
25
|
+
4. **韧性层是显式的** —— 5 分钟磁盘缓存、按错误类型分级的冷却(额度 1h / 反爬 10min / 超时 90s)、引擎健康上报,缺 key 的引擎提前剔除不占线程。
|
|
26
|
+
|
|
27
|
+
## 架构
|
|
28
|
+
|
|
29
|
+
```mermaid
|
|
30
|
+
flowchart LR
|
|
31
|
+
Q[query] --> X[query expansion<br/>local ollama]
|
|
32
|
+
X --> A[exa]
|
|
33
|
+
X --> B[brave]
|
|
34
|
+
X --> C[serper]
|
|
35
|
+
X --> D[deepseek]
|
|
36
|
+
X --> E[firecrawl]
|
|
37
|
+
X --> F[anysearch]
|
|
38
|
+
X --> G[tavily]
|
|
39
|
+
X --> H[searxng]
|
|
40
|
+
A & B & C & D & E & F & G & H --> R[RRF fusion<br/>top 15]
|
|
41
|
+
R --> J[jina rerank v3.5<br/>fallback v3 → v2]
|
|
42
|
+
J --> O[top N results<br/>+ provenance + health]
|
|
43
|
+
O --> AN[optional cited answer]
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## 安装
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/TsukimioRinka/agg-search
|
|
50
|
+
cd agg-search
|
|
51
|
+
python3 -m pip install -e .
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
只想要脚本、不想装包也行:`PYTHONPATH=src python3 -m agg_search.search "query"`。
|
|
55
|
+
|
|
56
|
+
## 使用
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
# 搜索
|
|
60
|
+
agg-search "rust async runtime comparison" --top-n 10 --recency month
|
|
61
|
+
|
|
62
|
+
# 机器可读输出
|
|
63
|
+
agg-search "MCP server examples" --json
|
|
64
|
+
|
|
65
|
+
# RAG-Fusion 查询扩展(需要本地 ollama)
|
|
66
|
+
agg-search "vector database tradeoffs" --expand 3
|
|
67
|
+
|
|
68
|
+
# 带引用的答案段(本地 ollama;--answer deepseek 走 API)
|
|
69
|
+
agg-search "what changed in Python 3.13" --answer
|
|
70
|
+
|
|
71
|
+
# 快速模式:单池扇出,最早两个有结果的引擎就返回
|
|
72
|
+
agg-search "breaking news" --fast
|
|
73
|
+
|
|
74
|
+
# 抓取正文:DeepSeek Fetch 优先,失败落容器阶梯
|
|
75
|
+
crawl-fetch --url https://example.com/post --tier auto --out md
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
密钥只从环境变量读(也兼容 `~/.config/agent-env`、`~/.config/ai-keys.env` 的 `KEY=` 行)。缺失的引擎会被跳过,不会让整次搜索失败。完整变量表见 [docs/environment.md](docs/environment.md)。
|
|
79
|
+
|
|
80
|
+
## MCP server
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"mcpServers": {
|
|
85
|
+
"agg-search": {
|
|
86
|
+
"command": "agg-search-mcp",
|
|
87
|
+
"env": { "BRAVE_API_KEY": "...", "EXA_API_KEY": "..." }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
暴露两个工具:
|
|
94
|
+
|
|
95
|
+
| 工具 | 参数 | 说明 |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| `search` | `query`(必填)、`top_n`、`recency`、`engines`、`expand`、`answer`、`fast`、`no_jina` | 并行召回 + RRF + 精排,返回 JSON |
|
|
98
|
+
| `fetch` | `urls`(必填)、`tier`、`out`、`text_limit` | 分层抓取正文 |
|
|
99
|
+
|
|
100
|
+
服务用换行分隔的 JSON-RPC 2.0,`initialize` / `tools/list` / `tools/call` / `ping` 都已实现,可直接挂到 MCP 客户端。
|
|
101
|
+
|
|
102
|
+
## 同类项目对比
|
|
103
|
+
|
|
104
|
+
只列能核实的能力,语言一列仅在仓库自述里明确时填写:
|
|
105
|
+
|
|
106
|
+
| 项目 | 栈 | 它强在哪 | 与本项目的差别 |
|
|
107
|
+
|---|---|---|---|
|
|
108
|
+
| [search-cli](https://github.com/paperfoot/search-cli) | Rust | 13 个 provider,单一二进制 | 本项目免编译;多容器抓取阶梯与 DeepSeek Fetch 是一等路径 |
|
|
109
|
+
| [web-search-mcp](https://github.com/charley-forey/web-search-mcp) | TypeScript | SSRF 安全抓取、注入防御(`<untrusted_content>` 包裹 + 隐藏字符剥离)、PDF/RSS/YouTube 抽取、本地语义重排、Recall@5 / MRR@10 评测、MCP resources/prompts | 它的抓取安全与内容抽取比本项目完整;本项目的差异在 8 引擎并行 + RRF→Jina listwise 两阶段排序、DeepSeek Search/Fetch、RAG-Fusion 扩展与容器化的静态→动态→反爬阶梯 |
|
|
110
|
+
| [argo](https://github.com/taxueseek/argo) | — | 300+ 来源、18 语言、MCP/CLI | 本项目只做通用网页检索,覆盖面窄;差异在排序链路与成本控制更细 |
|
|
111
|
+
| [websearch-mcpserver](https://github.com/daidaiJ/websearch-mcpserver) | Go | 零 key 可跑、多引擎 + 学术源 | 本项目按 key 选择性启用引擎,不承诺零 key |
|
|
112
|
+
| [free-search-api](https://github.com/Ethos6/free-search-api) | — | 自托管、无 key、Tavily 兼容端点 | 本项目是客户端编排层,不提供兼容 API |
|
|
113
|
+
|
|
114
|
+
## 测试
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
离线测试只覆盖纯函数(域名归并、去重、RRF、冷却分类、缓存键),不发起网络请求、不消耗额度。
|
|
121
|
+
|
|
122
|
+
## 安全与隐私
|
|
123
|
+
|
|
124
|
+
- 仓库内不含任何密钥、账号、内网地址;`.env.example` 只有空占位。
|
|
125
|
+
- 密钥只在进程环境里读取,不写日志、不落盘。
|
|
126
|
+
- 抓取在容器内执行,宿主机不留浏览器与爬虫依赖。
|
|
127
|
+
|
|
128
|
+
## License
|
|
129
|
+
|
|
130
|
+
MIT
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agg-search"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Multi-engine web search with RRF fusion, Jina reranking, query expansion and a tiered fetcher — CLI + MCP server, stdlib only."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "TsukimioRinka" }]
|
|
13
|
+
keywords = ["search", "mcp", "rrf", "rerank", "agents", "rag"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/TsukimioRinka/agg-search"
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
agg-search = "agg_search.search:main"
|
|
26
|
+
crawl-fetch = "agg_search.fetch:main"
|
|
27
|
+
agg-search-eval = "agg_search.eval:main"
|
|
28
|
+
agg-search-mcp = "agg_search.mcp:main"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools]
|
|
31
|
+
package-dir = { "" = "src" }
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
File without changes
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""RRF parameter sweep harness for agg-search — record once, sweep free.
|
|
3
|
+
|
|
4
|
+
record: run the judgment-set queries through the live engines ONCE and store
|
|
5
|
+
every engine's raw rows. This is the only step that spends engine
|
|
6
|
+
quota (Firecrawl credits, DeepSeek balance, ...).
|
|
7
|
+
sweep: replay the fixture entirely offline: RRF over a k x weights grid,
|
|
8
|
+
then (unless --no-jina) the real Jina rerank of each config's top-15,
|
|
9
|
+
scored against the relevance judgments. Jina tokens are the only
|
|
10
|
+
remaining cost (~2.5k tokens per config-query pair).
|
|
11
|
+
|
|
12
|
+
Usage:
|
|
13
|
+
python3 rrf_eval.py record --queries queries.json --out fixture.json
|
|
14
|
+
[--engines exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng]
|
|
15
|
+
[--per-engine 5]
|
|
16
|
+
python3 rrf_eval.py sweep --fixture fixture.json --judgments judgments.json
|
|
17
|
+
[--top 10] [--no-jina] [--workers 4] [--diversify]
|
|
18
|
+
|
|
19
|
+
queries.json: ["query one", "query two", ...]
|
|
20
|
+
judgments.json: {"query one": ["https://relevant-url", ...], ...}
|
|
21
|
+
Relevance labels are built by pooling every engine's results for the query
|
|
22
|
+
and judging title/snippet relevance (TREC-style pooling; labels are
|
|
23
|
+
model-made, not human gold).
|
|
24
|
+
"""
|
|
25
|
+
import argparse
|
|
26
|
+
import concurrent.futures as cf
|
|
27
|
+
import importlib.util
|
|
28
|
+
import json
|
|
29
|
+
import math
|
|
30
|
+
import sys
|
|
31
|
+
import time
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
HERE = Path(__file__).resolve().parent
|
|
35
|
+
|
|
36
|
+
try:
|
|
37
|
+
from . import search as m
|
|
38
|
+
except ImportError: # run as a plain script from the repository root
|
|
39
|
+
import importlib.util
|
|
40
|
+
|
|
41
|
+
spec = importlib.util.spec_from_file_location("agg_search", HERE / "search.py")
|
|
42
|
+
m = importlib.util.module_from_spec(spec)
|
|
43
|
+
spec.loader.exec_module(m)
|
|
44
|
+
|
|
45
|
+
K_VALUES = (0, 10, 30, 60, 100)
|
|
46
|
+
WEIGHT_VARIANTS = {
|
|
47
|
+
"current (searxng .5)": {"searxng": 0.5},
|
|
48
|
+
"flat 1.0": {"searxng": 1.0},
|
|
49
|
+
"searxng .3": {"searxng": 0.3},
|
|
50
|
+
"exa 1.5": {"exa": 1.5, "searxng": 0.5},
|
|
51
|
+
"deepseek 1.5": {"deepseek": 1.5, "searxng": 0.5},
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def record(a):
|
|
56
|
+
queries = json.loads(Path(a.queries).read_text(encoding="utf-8"))
|
|
57
|
+
engines = [e for e in a.engines.split(",") if e in m.ENGINES]
|
|
58
|
+
out = {"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
59
|
+
"per_engine": a.per_engine, "engines": engines, "queries": {}}
|
|
60
|
+
for q in queries:
|
|
61
|
+
results, status = m.recall(q, a.per_engine, engines, None)
|
|
62
|
+
out["queries"][q] = {"engines": results, "status": status}
|
|
63
|
+
print("recorded: %s | %s" % (q, "; ".join(
|
|
64
|
+
"%s=%s" % kv for kv in status.items())), file=sys.stderr)
|
|
65
|
+
Path(a.out).write_text(json.dumps(out, ensure_ascii=False, indent=1),
|
|
66
|
+
encoding="utf-8")
|
|
67
|
+
print("fixture -> %s" % a.out)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def ndcg10(order, relevant, top):
|
|
71
|
+
"""Binary-gain nDCG@top for one ranked list."""
|
|
72
|
+
dcg = sum(1.0 / math.log2(i + 2)
|
|
73
|
+
for i, u in enumerate(order[:top]) if u in relevant)
|
|
74
|
+
ideal = sum(1.0 / math.log2(i + 2) for i in range(min(len(relevant), top)))
|
|
75
|
+
return dcg / ideal if ideal else 0.0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def evaluate_config(fx, judgments, k, weights, top, use_jina, workers,
|
|
79
|
+
diversify=False):
|
|
80
|
+
"""One (k, weights) config over all queries; returns aggregate metrics."""
|
|
81
|
+
def one(item):
|
|
82
|
+
q, blob = item
|
|
83
|
+
ranked = m.rrf_fuse(blob["engines"], 15, weights, k=k,
|
|
84
|
+
diversify=diversify)
|
|
85
|
+
order15 = [u for u, _, _ in ranked]
|
|
86
|
+
rel = set(judgments.get(q, []))
|
|
87
|
+
rec15 = len(rel & set(order15)) / len(rel) if rel else 0.0
|
|
88
|
+
dom15 = len({m.domain_of(u) for u in order15})
|
|
89
|
+
if use_jina:
|
|
90
|
+
rr, _err = m.jina_rerank(q, ranked, top)
|
|
91
|
+
final = [u for u, _, _ in (rr if rr is not None else ranked)][:top]
|
|
92
|
+
else:
|
|
93
|
+
final = order15[:top]
|
|
94
|
+
return {"q": q, "recall15": rec15, "ndcg10": ndcg10(final, rel, top),
|
|
95
|
+
"hit5": 1.0 if rel & set(final[:5]) else 0.0,
|
|
96
|
+
"dom15": dom15,
|
|
97
|
+
"dom10": len({m.domain_of(u) for u in final})}
|
|
98
|
+
|
|
99
|
+
with cf.ThreadPoolExecutor(max_workers=workers) as ex:
|
|
100
|
+
per_query = list(ex.map(one, fx["queries"].items()))
|
|
101
|
+
n = len(per_query) or 1
|
|
102
|
+
return {"recall15": sum(x["recall15"] for x in per_query) / n,
|
|
103
|
+
"ndcg10": sum(x["ndcg10"] for x in per_query) / n,
|
|
104
|
+
"hit5": sum(x["hit5"] for x in per_query) / n,
|
|
105
|
+
"dom15": sum(x["dom15"] for x in per_query) / n,
|
|
106
|
+
"dom10": sum(x["dom10"] for x in per_query) / n,
|
|
107
|
+
"per_query": per_query}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def sweep(a):
|
|
111
|
+
fx = json.loads(Path(a.fixture).read_text(encoding="utf-8"))
|
|
112
|
+
judgments = json.loads(Path(a.judgments).read_text(encoding="utf-8"))
|
|
113
|
+
rows = []
|
|
114
|
+
for k in K_VALUES:
|
|
115
|
+
for name, w in WEIGHT_VARIANTS.items():
|
|
116
|
+
r = evaluate_config(fx, judgments, k, w, a.top,
|
|
117
|
+
not a.no_jina, a.workers,
|
|
118
|
+
diversify=a.diversify)
|
|
119
|
+
rows.append((k, name, r))
|
|
120
|
+
print("k=%-4s %-22s recall@15=%.3f nDCG@%s=%.3f hit@5=%.2f dom15=%.1f dom10=%.1f"
|
|
121
|
+
% (k, name, r["recall15"], a.top, r["ndcg10"], r["hit5"],
|
|
122
|
+
r["dom15"], r["dom10"]),
|
|
123
|
+
file=sys.stderr)
|
|
124
|
+
rows.sort(key=lambda x: (-x[2]["ndcg10"], -x[2]["recall15"]))
|
|
125
|
+
print("\n# RRF sweep — %d queries, Jina=%s, diversify=%s, top=%d"
|
|
126
|
+
% (len(fx["queries"]), "off" if a.no_jina else "v3.5",
|
|
127
|
+
"on" if a.diversify else "off", a.top))
|
|
128
|
+
print("\n| rank | k | weights | recall@15 | nDCG@%d | hit@5 | dom15 | dom10 |" % a.top)
|
|
129
|
+
print("|---|---|---|---|---|---|---|---|")
|
|
130
|
+
for i, (k, name, r) in enumerate(rows, 1):
|
|
131
|
+
print("| %d | %d | %s | %.3f | **%.3f** | %.2f | %.1f | %.1f |"
|
|
132
|
+
% (i, k, name, r["recall15"], r["ndcg10"], r["hit5"],
|
|
133
|
+
r["dom15"], r["dom10"]))
|
|
134
|
+
base = [r for k, name, r in rows if k == 60 and name.startswith("current")]
|
|
135
|
+
if base:
|
|
136
|
+
print("\ncurrent config (k=60, searxng .5): nDCG@%d=%.3f, recall@15=%.3f"
|
|
137
|
+
% (a.top, base[0]["ndcg10"], base[0]["recall15"]))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def main():
|
|
141
|
+
ap = argparse.ArgumentParser(description="RRF record/sweep harness")
|
|
142
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
143
|
+
r = sub.add_parser("record")
|
|
144
|
+
r.add_argument("--queries", required=True)
|
|
145
|
+
r.add_argument("--out", required=True)
|
|
146
|
+
r.add_argument("--engines",
|
|
147
|
+
default="exa,brave,serper,deepseek,firecrawl,anysearch,tavily,searxng")
|
|
148
|
+
r.add_argument("--per-engine", type=int, default=5)
|
|
149
|
+
s = sub.add_parser("sweep")
|
|
150
|
+
s.add_argument("--fixture", required=True)
|
|
151
|
+
s.add_argument("--judgments", required=True)
|
|
152
|
+
s.add_argument("--top", type=int, default=10)
|
|
153
|
+
s.add_argument("--no-jina", action="store_true")
|
|
154
|
+
s.add_argument("--workers", type=int, default=4)
|
|
155
|
+
s.add_argument("--diversify", action="store_true",
|
|
156
|
+
help="eTLD+1 interleave before top-15 cut (measures the layer)")
|
|
157
|
+
a = ap.parse_args()
|
|
158
|
+
if a.cmd == "record":
|
|
159
|
+
record(a)
|
|
160
|
+
else:
|
|
161
|
+
sweep(a)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
if __name__ == "__main__":
|
|
165
|
+
main()
|