bookfetch 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bookfetch-0.3.0/.gitignore +12 -0
- bookfetch-0.3.0/PKG-INFO +146 -0
- bookfetch-0.3.0/README.md +130 -0
- bookfetch-0.3.0/pyproject.toml +31 -0
- bookfetch-0.3.0/src/bookfetch/__init__.py +3 -0
- bookfetch-0.3.0/src/bookfetch/cli.py +169 -0
- bookfetch-0.3.0/src/bookfetch/model.py +64 -0
- bookfetch-0.3.0/src/bookfetch/sources/__init__.py +49 -0
- bookfetch-0.3.0/src/bookfetch/sources/base.py +24 -0
- bookfetch-0.3.0/src/bookfetch/sources/ctext.py +151 -0
- bookfetch-0.3.0/src/bookfetch/sources/github.py +152 -0
- bookfetch-0.3.0/src/bookfetch/sources/libgen.py +147 -0
- bookfetch-0.3.0/src/bookfetch/sources/wikisource.py +239 -0
- bookfetch-0.3.0/src/bookfetch/util/__init__.py +92 -0
- bookfetch-0.3.0/src/bookfetch/util/epub.py +156 -0
- bookfetch-0.3.0/src/bookfetch/util/simplify.py +19 -0
- bookfetch-0.3.0/src/bookfetch/util/splitters.py +72 -0
- bookfetch-0.3.0/tests/fixtures/README.md +14 -0
- bookfetch-0.3.0/tests/fixtures/chapter_296619_head.html +83 -0
- bookfetch-0.3.0/tests/fixtures/gbooks_tree.json +1 -0
- bookfetch-0.3.0/tests/fixtures/libgen_parked.html +18 -0
- bookfetch-0.3.0/tests/fixtures/res_book.html +85 -0
- bookfetch-0.3.0/tests/fixtures/searchbooks.html +75 -0
- bookfetch-0.3.0/tests/fixtures/ws_kuangren.html +223 -0
- bookfetch-0.3.0/tests/fixtures/ws_nahan_main.html +141 -0
- bookfetch-0.3.0/tests/test_ctext.py +46 -0
- bookfetch-0.3.0/tests/test_epub.py +68 -0
- bookfetch-0.3.0/tests/test_github.py +41 -0
- bookfetch-0.3.0/tests/test_libgen.py +32 -0
- bookfetch-0.3.0/tests/test_model.py +26 -0
- bookfetch-0.3.0/tests/test_simplify.py +27 -0
- bookfetch-0.3.0/tests/test_splitters.py +50 -0
- bookfetch-0.3.0/tests/test_util.py +19 -0
- bookfetch-0.3.0/tests/test_wikisource.py +65 -0
- bookfetch-0.3.0/uv.lock +200 -0
bookfetch-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bookfetch
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Agent-friendly ebook finder CLI: routes book queries to working sources (Chinese classics first).
|
|
5
|
+
Author-email: Helios Wei <helioswei1995@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: agent,chinese,classics,cli,ctext,ebook
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Provides-Extra: simp
|
|
14
|
+
Requires-Dist: opencc>=1.4; extra == 'simp'
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# bookfetch
|
|
18
|
+
|
|
19
|
+
给 Agent 用的电子书查找与下载 CLI:把书名丢给它,它自动把请求路由到**实际能下到书**的源。
|
|
20
|
+
|
|
21
|
+
设计出发点(实测结论):现成的 agent 找书技能几乎全部绑定 Z-Library / Libgen,而这两者对**中文古籍基本无效**(不收录 / 账号墙 / Cloudflare 墙)。中文古籍真正能用的源是 ctext.org(中国哲学书电子化计划:免费、带标点、国内直连)——没人把它做成 agent 工具,于是有了 bookfetch。
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
bookfetch search 渊海子平 # 跨源搜索,输出 JSON
|
|
25
|
+
bookfetch get ctext 727782 # 下载整本书到当前目录
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## 特性
|
|
29
|
+
|
|
30
|
+
- 书源路由:按书种/语言分发到可用源,单个源故障不影响整体(errors 独立上报)
|
|
31
|
+
- **EPUB / 章节感知**:`--format epub` 零依赖生成手机可读的 epub(含目录);`--split`
|
|
32
|
+
在 txt 中插入章节分隔;古籍《》/卷/序跋类标题行自动识别为章节
|
|
33
|
+
- JSON 优先输出:stdout 只吐结构化 JSON,agent 直接解析;`--human` 给人看
|
|
34
|
+
- 礼貌抓取:内置限速 + 重试退避 + 编码回退(GBK/Big5→UTF-8)
|
|
35
|
+
- **零运行时依赖**:纯 Python 标准库,任何环境装完即用(简体转换是可选扩展)
|
|
36
|
+
- 离线可测:解析测试基于真实抓包样本(fixtures),不依赖线上
|
|
37
|
+
|
|
38
|
+
## 安装
|
|
39
|
+
|
|
40
|
+
需要 Python >= 3.10。
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
# 推荐:uv
|
|
44
|
+
uv tool install git+https://github.com/Helioswei/bookfetch.git
|
|
45
|
+
|
|
46
|
+
# 或 pip
|
|
47
|
+
pip install git+https://github.com/Helioswei/bookfetch.git
|
|
48
|
+
|
|
49
|
+
# 本地开发
|
|
50
|
+
uv sync && uv run bookfetch search 论语
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## 用法
|
|
54
|
+
|
|
55
|
+
### 搜索
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
bookfetch search <书名>
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
输出(JSON,字段稳定,供 agent 消费):
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"cmd": "search",
|
|
66
|
+
"query": "渊海子平",
|
|
67
|
+
"results": [
|
|
68
|
+
{
|
|
69
|
+
"source": "ctext",
|
|
70
|
+
"id": "727782",
|
|
71
|
+
"title": "淵海子平",
|
|
72
|
+
"url": "https://ctext.org/wiki.pl?if=gb&res=727782",
|
|
73
|
+
"subtitle": "維基文字版:開放共同編輯的資料。",
|
|
74
|
+
"format_hint": "txt",
|
|
75
|
+
"extra": { "author": "徐子平" }
|
|
76
|
+
}
|
|
77
|
+
],
|
|
78
|
+
"count": 1,
|
|
79
|
+
"errors": {}
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`--source ctext` 限定源(可重复);`--limit N` 限制条数;`--human` 输出人类可读列表。
|
|
84
|
+
|
|
85
|
+
### 下载
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
bookfetch get ctext 727782 --out ./books # 默认 txt(整本合并)
|
|
89
|
+
bookfetch get ctext 727782 --format epub --out ./books # 手机友好的 epub(自动分章+目录)
|
|
90
|
+
bookfetch get ctext 727782 --split --out ./books # txt 中插入 === 章节 === 分隔
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
把 id 对应的整本书下载为 UTF-8 纯文本(ctext 的书会自动按序抓取全部章节并拼接)。
|
|
94
|
+
`epub` 与 `--split` 的章节来自源结构(ctext 分页)或《》/卷/序跋类标题行自动识别;
|
|
95
|
+
正文一字不改,标题行仅在阅读视图去重。
|
|
96
|
+
|
|
97
|
+
#### 繁转简(可选)
|
|
98
|
+
|
|
99
|
+
默认保留古籍繁体原文;需要简体版时加 `--simplify`(需先装可选依赖):
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
uv tool install bookfetch --extra simp # 或 pip install 'bookfetch[simp]'
|
|
103
|
+
bookfetch get ctext 727782 --out ./books --simplify
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
转换基于 OpenCC(t2s),文件与文件名会一并转为简体。古籍存在异体字/通假字,
|
|
107
|
+
转换非 100% 保真,学术用途请以原文为准。
|
|
108
|
+
|
|
109
|
+
## 已支持的书源
|
|
110
|
+
|
|
111
|
+
| 源 | 覆盖 | 说明 |
|
|
112
|
+
|---|---|---|
|
|
113
|
+
| ctext | 中文古籍(免费全文、带标点) | 书目检索 + 多章节整本下载 |
|
|
114
|
+
| github | 公版中文古籍文本仓库 | 精选仓库树索引(7 天缓存),raw 直连下载 |
|
|
115
|
+
| wikisource | 中文/英文公版书(含现代公版:鲁迅等) | MediaWiki API + 渲染页解析,目录自动展开整本;大陆访问需代理 |
|
|
116
|
+
| libgen | 英文现代书(原文件 epub/pdf) | 探活式镜像链(域名轮换频繁),当前镜像不可达时会明确报错;需代理 |
|
|
117
|
+
|
|
118
|
+
> 网络提示:ctext/github 大陆直连可用;wikisource(Wikimedia)与 libgen 大陆直连不通,
|
|
119
|
+
> 需能访问对应站点的网络环境(如代理),本工具遵循系统 http_proxy/https_proxy 环境变量。
|
|
120
|
+
|
|
121
|
+
## 源与合规
|
|
122
|
+
|
|
123
|
+
- bookfetch 是**路由与下载工具**,不存储、不重新分发任何书籍内容;下载物只落在使用者本地
|
|
124
|
+
- 各源内容版权归原作者/整理者所有。公版内容可自由使用;**仍在版权期内的内容,请使用者自行确认下载与使用的合法性**
|
|
125
|
+
- 抓取行为遵守各源访问条款:公开页面、内置限速、不做任何绕过(登录墙/验证码/反爬规避)
|
|
126
|
+
- 测试 fixture 为各源页面/结构的极小样本,仅用于解析测试,来源记录见 tests/fixtures/README.md
|
|
127
|
+
- 任何权利方认为本工具对某源的使用不妥,请提 issue,我们会调整或移除该源
|
|
128
|
+
|
|
129
|
+
## 开发与测试
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
uv sync --group dev
|
|
133
|
+
uv run pytest -q # 离线测试,基于 tests/fixtures 真实抓包样本
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## 路线图
|
|
137
|
+
|
|
138
|
+
- [x] M1: ctext 源 + search/get CLI(2026-09-03 完成)
|
|
139
|
+
- [x] M2: github 古籍源 + OpenCC 简繁转换 + 合规声明(2026-09-03 完成)
|
|
140
|
+
- [x] M3: EPUB 转换 + 章节切分(2026-09-04 完成,零依赖手写 zip+xhtml)
|
|
141
|
+
- [x] M4: wikisource 中/英公版源 + libgen 探活镜像链 + 白话注解 spike(2026-09-04 完成;白话注解判定放弃,见 PRD)
|
|
142
|
+
- [ ] 规划中: SKILL.md agent 外壳 + PyPI 发布
|
|
143
|
+
|
|
144
|
+
## 许可
|
|
145
|
+
|
|
146
|
+
MIT。只面向公版/开放文本(ctext 收录均为公版古籍)。请遵守各源的访问条款。
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# bookfetch
|
|
2
|
+
|
|
3
|
+
给 Agent 用的电子书查找与下载 CLI:把书名丢给它,它自动把请求路由到**实际能下到书**的源。
|
|
4
|
+
|
|
5
|
+
设计出发点(实测结论):现成的 agent 找书技能几乎全部绑定 Z-Library / Libgen,而这两者对**中文古籍基本无效**(不收录 / 账号墙 / Cloudflare 墙)。中文古籍真正能用的源是 ctext.org(中国哲学书电子化计划:免费、带标点、国内直连)——没人把它做成 agent 工具,于是有了 bookfetch。
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
bookfetch search 渊海子平 # 跨源搜索,输出 JSON
|
|
9
|
+
bookfetch get ctext 727782 # 下载整本书到当前目录
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## 特性
|
|
13
|
+
|
|
14
|
+
- 书源路由:按书种/语言分发到可用源,单个源故障不影响整体(errors 独立上报)
|
|
15
|
+
- **EPUB / 章节感知**:`--format epub` 零依赖生成手机可读的 epub(含目录);`--split`
|
|
16
|
+
在 txt 中插入章节分隔;古籍《》/卷/序跋类标题行自动识别为章节
|
|
17
|
+
- JSON 优先输出:stdout 只吐结构化 JSON,agent 直接解析;`--human` 给人看
|
|
18
|
+
- 礼貌抓取:内置限速 + 重试退避 + 编码回退(GBK/Big5→UTF-8)
|
|
19
|
+
- **零运行时依赖**:纯 Python 标准库,任何环境装完即用(简体转换是可选扩展)
|
|
20
|
+
- 离线可测:解析测试基于真实抓包样本(fixtures),不依赖线上
|
|
21
|
+
|
|
22
|
+
## 安装
|
|
23
|
+
|
|
24
|
+
需要 Python >= 3.10。
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# 推荐:uv
|
|
28
|
+
uv tool install git+https://github.com/Helioswei/bookfetch.git
|
|
29
|
+
|
|
30
|
+
# 或 pip
|
|
31
|
+
pip install git+https://github.com/Helioswei/bookfetch.git
|
|
32
|
+
|
|
33
|
+
# 本地开发
|
|
34
|
+
uv sync && uv run bookfetch search 论语
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## 用法
|
|
38
|
+
|
|
39
|
+
### 搜索
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
bookfetch search <书名>
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
输出(JSON,字段稳定,供 agent 消费):
|
|
46
|
+
|
|
47
|
+
```json
|
|
48
|
+
{
|
|
49
|
+
"cmd": "search",
|
|
50
|
+
"query": "渊海子平",
|
|
51
|
+
"results": [
|
|
52
|
+
{
|
|
53
|
+
"source": "ctext",
|
|
54
|
+
"id": "727782",
|
|
55
|
+
"title": "淵海子平",
|
|
56
|
+
"url": "https://ctext.org/wiki.pl?if=gb&res=727782",
|
|
57
|
+
"subtitle": "維基文字版:開放共同編輯的資料。",
|
|
58
|
+
"format_hint": "txt",
|
|
59
|
+
"extra": { "author": "徐子平" }
|
|
60
|
+
}
|
|
61
|
+
],
|
|
62
|
+
"count": 1,
|
|
63
|
+
"errors": {}
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`--source ctext` 限定源(可重复);`--limit N` 限制条数;`--human` 输出人类可读列表。
|
|
68
|
+
|
|
69
|
+
### 下载
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
bookfetch get ctext 727782 --out ./books # 默认 txt(整本合并)
|
|
73
|
+
bookfetch get ctext 727782 --format epub --out ./books # 手机友好的 epub(自动分章+目录)
|
|
74
|
+
bookfetch get ctext 727782 --split --out ./books # txt 中插入 === 章节 === 分隔
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
把 id 对应的整本书下载为 UTF-8 纯文本(ctext 的书会自动按序抓取全部章节并拼接)。
|
|
78
|
+
`epub` 与 `--split` 的章节来自源结构(ctext 分页)或《》/卷/序跋类标题行自动识别;
|
|
79
|
+
正文一字不改,标题行仅在阅读视图去重。
|
|
80
|
+
|
|
81
|
+
#### 繁转简(可选)
|
|
82
|
+
|
|
83
|
+
默认保留古籍繁体原文;需要简体版时加 `--simplify`(需先装可选依赖):
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv tool install bookfetch --extra simp # 或 pip install 'bookfetch[simp]'
|
|
87
|
+
bookfetch get ctext 727782 --out ./books --simplify
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
转换基于 OpenCC(t2s),文件与文件名会一并转为简体。古籍存在异体字/通假字,
|
|
91
|
+
转换非 100% 保真,学术用途请以原文为准。
|
|
92
|
+
|
|
93
|
+
## 已支持的书源
|
|
94
|
+
|
|
95
|
+
| 源 | 覆盖 | 说明 |
|
|
96
|
+
|---|---|---|
|
|
97
|
+
| ctext | 中文古籍(免费全文、带标点) | 书目检索 + 多章节整本下载 |
|
|
98
|
+
| github | 公版中文古籍文本仓库 | 精选仓库树索引(7 天缓存),raw 直连下载 |
|
|
99
|
+
| wikisource | 中文/英文公版书(含现代公版:鲁迅等) | MediaWiki API + 渲染页解析,目录自动展开整本;大陆访问需代理 |
|
|
100
|
+
| libgen | 英文现代书(原文件 epub/pdf) | 探活式镜像链(域名轮换频繁),当前镜像不可达时会明确报错;需代理 |
|
|
101
|
+
|
|
102
|
+
> 网络提示:ctext/github 大陆直连可用;wikisource(Wikimedia)与 libgen 大陆直连不通,
|
|
103
|
+
> 需能访问对应站点的网络环境(如代理),本工具遵循系统 http_proxy/https_proxy 环境变量。
|
|
104
|
+
|
|
105
|
+
## 源与合规
|
|
106
|
+
|
|
107
|
+
- bookfetch 是**路由与下载工具**,不存储、不重新分发任何书籍内容;下载物只落在使用者本地
|
|
108
|
+
- 各源内容版权归原作者/整理者所有。公版内容可自由使用;**仍在版权期内的内容,请使用者自行确认下载与使用的合法性**
|
|
109
|
+
- 抓取行为遵守各源访问条款:公开页面、内置限速、不做任何绕过(登录墙/验证码/反爬规避)
|
|
110
|
+
- 测试 fixture 为各源页面/结构的极小样本,仅用于解析测试,来源记录见 tests/fixtures/README.md
|
|
111
|
+
- 任何权利方认为本工具对某源的使用不妥,请提 issue,我们会调整或移除该源
|
|
112
|
+
|
|
113
|
+
## 开发与测试
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
uv sync --group dev
|
|
117
|
+
uv run pytest -q # 离线测试,基于 tests/fixtures 真实抓包样本
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## 路线图
|
|
121
|
+
|
|
122
|
+
- [x] M1: ctext 源 + search/get CLI(2026-09-03 完成)
|
|
123
|
+
- [x] M2: github 古籍源 + OpenCC 简繁转换 + 合规声明(2026-09-03 完成)
|
|
124
|
+
- [x] M3: EPUB 转换 + 章节切分(2026-09-04 完成,零依赖手写 zip+xhtml)
|
|
125
|
+
- [x] M4: wikisource 中/英公版源 + libgen 探活镜像链 + 白话注解 spike(2026-09-04 完成;白话注解判定放弃,见 PRD)
|
|
126
|
+
- [ ] 规划中: SKILL.md agent 外壳 + PyPI 发布
|
|
127
|
+
|
|
128
|
+
## 许可
|
|
129
|
+
|
|
130
|
+
MIT。只面向公版/开放文本(ctext 收录均为公版古籍)。请遵守各源的访问条款。
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "bookfetch"
|
|
3
|
+
version = "0.3.0"
|
|
4
|
+
description = "Agent-friendly ebook finder CLI: routes book queries to working sources (Chinese classics first)."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [{ name = "Helios Wei", email = "helioswei1995@gmail.com" }]
|
|
9
|
+
keywords = ["ebook", "chinese", "classics", "ctext", "cli", "agent"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Environment :: Console",
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"License :: OSI Approved :: MIT License",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.scripts]
|
|
18
|
+
bookfetch = "bookfetch.cli:main"
|
|
19
|
+
|
|
20
|
+
[build-system]
|
|
21
|
+
requires = ["hatchling"]
|
|
22
|
+
build-backend = "hatchling.build"
|
|
23
|
+
|
|
24
|
+
[tool.hatch.build.targets.wheel]
|
|
25
|
+
packages = ["src/bookfetch"]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
simp = ["opencc>=1.4"]
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = ["pytest>=8.0", "opencc>=1.4"]
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""bookfetch CLI — JSON-first output for agents, --human for people."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .model import Book, Chapter, FetchResult
|
|
12
|
+
from .sources import get_source, search_all, source_names
|
|
13
|
+
from .util import FetchError, sanitize_filename
|
|
14
|
+
from .util.epub import build_epub
|
|
15
|
+
from .util.simplify import to_simplified
|
|
16
|
+
from .util.splitters import split_headings
|
|
17
|
+
|
|
18
|
+
DESC = "Agent-friendly ebook finder: routes book queries to working sources."
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _human_search(obj: dict) -> None:
|
|
22
|
+
for i, b in enumerate(obj["results"], 1):
|
|
23
|
+
extra = f" [{b['subtitle']}]" if b.get("subtitle") else ""
|
|
24
|
+
print(f"{i}. [{b['source']}] {b['title']} (id={b['id']}){extra}")
|
|
25
|
+
if obj.get("errors"):
|
|
26
|
+
for src, err in obj["errors"].items():
|
|
27
|
+
print(f" ! {src}: {err}", file=sys.stderr)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _human_get(obj: dict) -> None:
|
|
31
|
+
r = obj["result"]
|
|
32
|
+
print(f"Saved: {r['out_path']}")
|
|
33
|
+
extra = f" | {len(r['chapters'])} chapters" if r.get("chapters") else ""
|
|
34
|
+
print(f" {r['title']} | {r['lines']} paragraphs | {r['chars']} chars | {r['format']}{extra}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _ensure_chapters(fr: FetchResult) -> list[Chapter]:
|
|
38
|
+
"""Chapters from the source, or heading-split, or one whole-text chapter."""
|
|
39
|
+
if fr.chapters:
|
|
40
|
+
return list(fr.chapters)
|
|
41
|
+
chs = split_headings(fr.content.splitlines())
|
|
42
|
+
if chs:
|
|
43
|
+
return chs
|
|
44
|
+
return [Chapter(title=fr.title or fr.id, text=fr.content)]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _render_get(src, book: Book, args) -> FetchResult:
|
|
48
|
+
"""Fetch -> optional simplify -> render txt/epub under --out.
|
|
49
|
+
|
|
50
|
+
Binary sources (libgen etc.) return FetchResult.raw and are saved
|
|
51
|
+
byte-for-byte; text-only flags (--simplify/--split/--format) reject them.
|
|
52
|
+
"""
|
|
53
|
+
fr = src.fetch(book)
|
|
54
|
+
|
|
55
|
+
if fr.raw is not None: # binary passthrough: save the original file
|
|
56
|
+
if args.simplify or args.split or args.format != "txt":
|
|
57
|
+
raise ValueError(f"源 {fr.source} 是二进制原文件(.{fr.format}),不支持 --simplify/--split/--format")
|
|
58
|
+
fname = sanitize_filename(fr.title) or fr.id
|
|
59
|
+
out_dir = Path(args.out)
|
|
60
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
61
|
+
path = out_dir / f"{fname}.{fr.format}"
|
|
62
|
+
path.write_bytes(fr.raw)
|
|
63
|
+
fr.out_path = str(path)
|
|
64
|
+
return fr
|
|
65
|
+
|
|
66
|
+
chapters = _ensure_chapters(fr)
|
|
67
|
+
|
|
68
|
+
if args.simplify:
|
|
69
|
+
chapters = [Chapter(title=to_simplified(c.title), text=to_simplified(c.text)) for c in chapters]
|
|
70
|
+
fr.title = to_simplified(fr.title)
|
|
71
|
+
|
|
72
|
+
fname = sanitize_filename(fr.title) or fr.id
|
|
73
|
+
out_dir = Path(args.out)
|
|
74
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
75
|
+
|
|
76
|
+
if args.format == "epub":
|
|
77
|
+
path = build_epub(fr.title, chapters, out_dir / f"{fname}.epub")
|
|
78
|
+
text_chars = sum(len(c.text) for c in chapters)
|
|
79
|
+
n_lines = sum(len(c.text.splitlines()) for c in chapters)
|
|
80
|
+
else: # txt
|
|
81
|
+
if args.split and len(chapters) > 1:
|
|
82
|
+
parts = []
|
|
83
|
+
for i, c in enumerate(chapters, 1):
|
|
84
|
+
head = c.title or f"第{i}部分"
|
|
85
|
+
parts.append(f"=== {head} ===\n{c.text}")
|
|
86
|
+
text = "\n\n".join(parts) + "\n"
|
|
87
|
+
elif args.simplify:
|
|
88
|
+
text = "\n".join(c.text for c in chapters) + "\n"
|
|
89
|
+
else:
|
|
90
|
+
text = fr.content # byte-identical to the fetched text
|
|
91
|
+
path = out_dir / f"{fname}.txt"
|
|
92
|
+
path.write_text(text, encoding="utf-8")
|
|
93
|
+
text_chars = len(text)
|
|
94
|
+
n_lines = len(text.splitlines())
|
|
95
|
+
|
|
96
|
+
fr.out_path = str(path)
|
|
97
|
+
fr.format = args.format
|
|
98
|
+
fr.chars = text_chars
|
|
99
|
+
fr.lines = n_lines
|
|
100
|
+
return fr
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def main(argv: list[str] | None = None) -> int:
|
|
104
|
+
p = argparse.ArgumentParser(prog="bookfetch", description=DESC)
|
|
105
|
+
p.add_argument("--version", action="version", version=f"bookfetch {__version__}")
|
|
106
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
107
|
+
|
|
108
|
+
sp = sub.add_parser("search", help="search sources for a book")
|
|
109
|
+
sp.add_argument("query", help="book title or keywords (Chinese OK)")
|
|
110
|
+
sp.add_argument("--source", action="append", default=None, help="only search this source (repeatable)")
|
|
111
|
+
sp.add_argument("--limit", type=int, default=20, help="max results (default 20)")
|
|
112
|
+
sp.add_argument("--human", action="store_true", help="human-readable output")
|
|
113
|
+
|
|
114
|
+
gp = sub.add_parser("get", help="download a book edition (id from search)")
|
|
115
|
+
gp.add_argument("source", help="source name, e.g. ctext")
|
|
116
|
+
gp.add_argument("id", help="edition id from search results")
|
|
117
|
+
gp.add_argument("--title", default="", help="optional title override for the output filename")
|
|
118
|
+
gp.add_argument("--out", default=".", help="output directory (default: current dir)")
|
|
119
|
+
gp.add_argument(
|
|
120
|
+
"--format",
|
|
121
|
+
choices=["txt", "epub"],
|
|
122
|
+
default="txt",
|
|
123
|
+
help="output format (default: txt; epub needs no extra deps)",
|
|
124
|
+
)
|
|
125
|
+
gp.add_argument(
|
|
126
|
+
"--split",
|
|
127
|
+
action="store_true",
|
|
128
|
+
help="insert '=== 章节 ===' separators into txt output (epub is always split)",
|
|
129
|
+
)
|
|
130
|
+
gp.add_argument(
|
|
131
|
+
"--simplify",
|
|
132
|
+
action="store_true",
|
|
133
|
+
help="convert Traditional Chinese to Simplified (requires the [simp] extra: OpenCC)",
|
|
134
|
+
)
|
|
135
|
+
gp.add_argument("--human", action="store_true", help="human-readable output")
|
|
136
|
+
|
|
137
|
+
args = p.parse_args(argv)
|
|
138
|
+
try:
|
|
139
|
+
if args.cmd == "search":
|
|
140
|
+
results, errors = search_all(args.query, args.source, args.limit)
|
|
141
|
+
obj = {
|
|
142
|
+
"cmd": "search",
|
|
143
|
+
"query": args.query,
|
|
144
|
+
"results": [b.to_dict() for b in results],
|
|
145
|
+
"count": len(results),
|
|
146
|
+
"errors": errors,
|
|
147
|
+
}
|
|
148
|
+
else: # get
|
|
149
|
+
src = get_source(args.source)
|
|
150
|
+
if src is None:
|
|
151
|
+
raise ValueError(f"unknown source {args.source!r} (known: {', '.join(source_names())})")
|
|
152
|
+
book = Book(source=args.source, id=args.id, title=args.title)
|
|
153
|
+
fr = _render_get(src, book, args)
|
|
154
|
+
obj = {"cmd": "get", "result": fr.to_dict()}
|
|
155
|
+
|
|
156
|
+
if getattr(args, "human", False) and args.cmd == "search":
|
|
157
|
+
_human_search(obj)
|
|
158
|
+
elif getattr(args, "human", False) and args.cmd == "get":
|
|
159
|
+
_human_get(obj)
|
|
160
|
+
else:
|
|
161
|
+
print(json.dumps(obj, ensure_ascii=False))
|
|
162
|
+
return 0
|
|
163
|
+
except (ValueError, FetchError) as e:
|
|
164
|
+
print(json.dumps({"cmd": getattr(args, "cmd", None), "error": str(e)}, ensure_ascii=False))
|
|
165
|
+
return 1
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
if __name__ == "__main__":
|
|
169
|
+
sys.exit(main())
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Shared result models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import asdict, dataclass, field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class Book:
|
|
10
|
+
"""One downloadable edition found by a source."""
|
|
11
|
+
|
|
12
|
+
source: str
|
|
13
|
+
id: str
|
|
14
|
+
title: str
|
|
15
|
+
url: str = ""
|
|
16
|
+
subtitle: str = ""
|
|
17
|
+
format_hint: str = "txt"
|
|
18
|
+
extra: dict = field(default_factory=dict)
|
|
19
|
+
|
|
20
|
+
def to_dict(self) -> dict:
|
|
21
|
+
return asdict(self)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Chapter:
|
|
26
|
+
"""A titled section of a fetched book (source pages, 《》 headings, ...)."""
|
|
27
|
+
|
|
28
|
+
title: str
|
|
29
|
+
text: str
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class FetchResult:
|
|
34
|
+
"""Parsed content of a downloaded Book.
|
|
35
|
+
|
|
36
|
+
Sources return content + optional chapter structure WITHOUT writing files;
|
|
37
|
+
the CLI renders the requested format (txt/epub) and sets ``out_path``.
|
|
38
|
+
``content`` is the plain merged text: joining each chapter's ``text`` with
|
|
39
|
+
``"\\n"`` reproduces it exactly (chapters are ordered slices of lines), so
|
|
40
|
+
format rendering never loses or reorders anything.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
source: str
|
|
44
|
+
id: str
|
|
45
|
+
title: str
|
|
46
|
+
out_path: str = ""
|
|
47
|
+
chars: int = 0
|
|
48
|
+
lines: int = 0
|
|
49
|
+
format: str = "txt"
|
|
50
|
+
content: str = ""
|
|
51
|
+
chapters: list[Chapter] | None = None
|
|
52
|
+
raw: bytes | None = None # binary sources (libgen): file bytes, no text pipeline
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> dict:
|
|
55
|
+
return {
|
|
56
|
+
"source": self.source,
|
|
57
|
+
"id": self.id,
|
|
58
|
+
"title": self.title,
|
|
59
|
+
"out_path": self.out_path,
|
|
60
|
+
"chars": self.chars,
|
|
61
|
+
"lines": self.lines,
|
|
62
|
+
"format": self.format,
|
|
63
|
+
"chapters": [c.title for c in self.chapters] if self.chapters else None,
|
|
64
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Source registry."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from ..model import Book
|
|
6
|
+
from .base import Source
|
|
7
|
+
from .ctext import Ctext
|
|
8
|
+
from .github import GithubBooks
|
|
9
|
+
from .libgen import Libgen
|
|
10
|
+
from .wikisource import Wikisource
|
|
11
|
+
|
|
12
|
+
_REGISTRY: dict[str, Source] = {}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _register(src: Source) -> Source:
|
|
16
|
+
_REGISTRY[src.name] = src
|
|
17
|
+
return src
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
_register(Ctext())
|
|
21
|
+
_register(GithubBooks())
|
|
22
|
+
_register(Wikisource("zh"))
|
|
23
|
+
_register(Wikisource("en"))
|
|
24
|
+
_register(Libgen())
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_source(name: str) -> Source | None:
|
|
28
|
+
return _REGISTRY.get(name)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def source_names() -> list[str]:
|
|
32
|
+
return list(_REGISTRY)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def search_all(query: str, names: list[str] | None = None, limit: int = 20):
|
|
36
|
+
"""Search across sources. Returns (results, errors_by_source)."""
|
|
37
|
+
names = names or source_names()
|
|
38
|
+
results: list[Book] = []
|
|
39
|
+
errors: dict[str, str] = {}
|
|
40
|
+
for n in names:
|
|
41
|
+
src = _REGISTRY.get(n)
|
|
42
|
+
if src is None:
|
|
43
|
+
errors[n] = f"unknown source (known: {', '.join(source_names())})"
|
|
44
|
+
continue
|
|
45
|
+
try:
|
|
46
|
+
results.extend(src.search(query))
|
|
47
|
+
except Exception as e: # source failure must not kill the whole search
|
|
48
|
+
errors[n] = f"{type(e).__name__}: {e}"
|
|
49
|
+
return results[:limit], errors
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Source adapter interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
from ..model import Book, FetchResult
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Source(ABC):
|
|
11
|
+
"""One book source (ctext, github, ...). Implementations must be stateless
|
|
12
|
+
except for util's built-in rate limiting."""
|
|
13
|
+
|
|
14
|
+
name: str = "base"
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def search(self, query: str) -> list[Book]:
|
|
18
|
+
"""Return editions matching query. Never raises for network issues —
|
|
19
|
+
callers surface errors via the errors dict instead."""
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def fetch(self, book: Book) -> FetchResult:
|
|
23
|
+
"""Fetch and parse one edition into content + optional chapter
|
|
24
|
+
structure. Never writes files — the CLI renders txt/epub."""
|