cnkibug 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cnkibug-0.7.0/LICENSE +21 -0
- cnkibug-0.7.0/PKG-INFO +84 -0
- cnkibug-0.7.0/README.md +233 -0
- cnkibug-0.7.0/cnkibug/__init__.py +0 -0
- cnkibug-0.7.0/cnkibug/app/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/app/cli.py +231 -0
- cnkibug-0.7.0/cnkibug/app/console.py +27 -0
- cnkibug-0.7.0/cnkibug/app/environment.py +47 -0
- cnkibug-0.7.0/cnkibug/app/errors.py +38 -0
- cnkibug-0.7.0/cnkibug/app/events.py +181 -0
- cnkibug-0.7.0/cnkibug/app/prompts.py +305 -0
- cnkibug-0.7.0/cnkibug/app/report_view.py +107 -0
- cnkibug-0.7.0/cnkibug/app/runtime.py +350 -0
- cnkibug-0.7.0/cnkibug/app/ui.py +322 -0
- cnkibug-0.7.0/cnkibug/browser/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/browser/cache.py +121 -0
- cnkibug-0.7.0/cnkibug/browser/runtime.py +92 -0
- cnkibug-0.7.0/cnkibug/browser/session.py +68 -0
- cnkibug-0.7.0/cnkibug/cnki/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/cnki/advanced.py +89 -0
- cnkibug-0.7.0/cnkibug/cnki/citation.py +68 -0
- cnkibug-0.7.0/cnkibug/cnki/details.py +181 -0
- cnkibug-0.7.0/cnkibug/cnki/guard.py +150 -0
- cnkibug-0.7.0/cnkibug/cnki/keyword.py +382 -0
- cnkibug-0.7.0/cnkibug/cnki/metrics.py +54 -0
- cnkibug-0.7.0/cnkibug/cnki/models.py +65 -0
- cnkibug-0.7.0/cnkibug/cnki/pages.py +629 -0
- cnkibug-0.7.0/cnkibug/cnki/pagination.py +160 -0
- cnkibug-0.7.0/cnkibug/cnki/results.py +173 -0
- cnkibug-0.7.0/cnkibug/cnki/resume.py +116 -0
- cnkibug-0.7.0/cnkibug/cnki/search.py +399 -0
- cnkibug-0.7.0/cnkibug/cnki/selectors.py +46 -0
- cnkibug-0.7.0/cnkibug/core/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/core/estimate.py +134 -0
- cnkibug-0.7.0/cnkibug/core/events.py +24 -0
- cnkibug-0.7.0/cnkibug/core/memory.py +150 -0
- cnkibug-0.7.0/cnkibug/core/runtime.py +14 -0
- cnkibug-0.7.0/cnkibug/core/search_query.py +104 -0
- cnkibug-0.7.0/cnkibug/core/settings.py +40 -0
- cnkibug-0.7.0/cnkibug/core/version.py +30 -0
- cnkibug-0.7.0/cnkibug/fileio/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/fileio/exporter.py +547 -0
- cnkibug-0.7.0/cnkibug/fileio/keyword_input.py +90 -0
- cnkibug-0.7.0/cnkibug/fileio/paths.py +84 -0
- cnkibug-0.7.0/cnkibug/gui/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/gui/advanced.py +278 -0
- cnkibug-0.7.0/cnkibug/gui/app.py +1506 -0
- cnkibug-0.7.0/cnkibug/gui/apply_update.ps1 +68 -0
- cnkibug-0.7.0/cnkibug/gui/events.py +45 -0
- cnkibug-0.7.0/cnkibug/gui/settings.py +234 -0
- cnkibug-0.7.0/cnkibug/gui/update_dialog.py +232 -0
- cnkibug-0.7.0/cnkibug/gui/updater.py +314 -0
- cnkibug-0.7.0/cnkibug/launcher.py +36 -0
- cnkibug-0.7.0/cnkibug/workflow/__init__.py +1 -0
- cnkibug-0.7.0/cnkibug/workflow/finalize.py +193 -0
- cnkibug-0.7.0/cnkibug/workflow/keyword_run.py +428 -0
- cnkibug-0.7.0/cnkibug/workflow/report.py +296 -0
- cnkibug-0.7.0/cnkibug/workflow/runner.py +161 -0
- cnkibug-0.7.0/cnkibug/workflow/state.py +332 -0
- cnkibug-0.7.0/cnkibug/workflow/task.py +164 -0
- cnkibug-0.7.0/cnkibug.egg-info/PKG-INFO +84 -0
- cnkibug-0.7.0/cnkibug.egg-info/SOURCES.txt +86 -0
- cnkibug-0.7.0/cnkibug.egg-info/dependency_links.txt +1 -0
- cnkibug-0.7.0/cnkibug.egg-info/entry_points.txt +5 -0
- cnkibug-0.7.0/cnkibug.egg-info/requires.txt +13 -0
- cnkibug-0.7.0/cnkibug.egg-info/top_level.txt +1 -0
- cnkibug-0.7.0/docs/pypi.md +43 -0
- cnkibug-0.7.0/pyproject.toml +40 -0
- cnkibug-0.7.0/setup.cfg +4 -0
- cnkibug-0.7.0/tests/test_advanced_search.py +179 -0
- cnkibug-0.7.0/tests/test_citation_fetcher.py +124 -0
- cnkibug-0.7.0/tests/test_cli.py +71 -0
- cnkibug-0.7.0/tests/test_cnki_details.py +277 -0
- cnkibug-0.7.0/tests/test_cnki_pagination.py +43 -0
- cnkibug-0.7.0/tests/test_cnki_records.py +194 -0
- cnkibug-0.7.0/tests/test_dom_contract.py +71 -0
- cnkibug-0.7.0/tests/test_exporter.py +486 -0
- cnkibug-0.7.0/tests/test_gui.py +369 -0
- cnkibug-0.7.0/tests/test_gui_updater.py +236 -0
- cnkibug-0.7.0/tests/test_keyword_import.py +46 -0
- cnkibug-0.7.0/tests/test_keyword_scraper.py +961 -0
- cnkibug-0.7.0/tests/test_launcher.py +16 -0
- cnkibug-0.7.0/tests/test_prompts.py +60 -0
- cnkibug-0.7.0/tests/test_runtime.py +205 -0
- cnkibug-0.7.0/tests/test_scrape_report.py +74 -0
- cnkibug-0.7.0/tests/test_scrape_workflow.py +410 -0
- cnkibug-0.7.0/tests/test_session_cache.py +45 -0
- cnkibug-0.7.0/tests/test_task_state.py +266 -0
cnkibug-0.7.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kaffu_Alcaid
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
cnkibug-0.7.0/PKG-INFO
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cnkibug
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: CNKI literature metadata collection with a desktop GUI
|
|
5
|
+
License: MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2026 Kaffu_Alcaid
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
|
|
27
|
+
Project-URL: Repository, https://github.com/KaffuAlcaid/CNKIBug
|
|
28
|
+
Project-URL: Issues, https://github.com/KaffuAlcaid/CNKIBug/issues
|
|
29
|
+
Requires-Python: <3.15,>=3.10
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Requires-Dist: playwright==1.60.0
|
|
33
|
+
Requires-Dist: openpyxl==3.1.5
|
|
34
|
+
Requires-Dist: pywin32==311; sys_platform == "win32"
|
|
35
|
+
Requires-Dist: psutil==7.2.2
|
|
36
|
+
Requires-Dist: rich==15.0.0
|
|
37
|
+
Requires-Dist: tomli>=2.0; python_version < "3.11"
|
|
38
|
+
Provides-Extra: gui
|
|
39
|
+
Requires-Dist: ttkbootstrap==1.20.4; extra == "gui"
|
|
40
|
+
Dynamic: license-file
|
|
41
|
+
|
|
42
|
+
# CNKIBug
|
|
43
|
+
|
|
44
|
+
中国知网论文信息抓取工具,提供 GUI 和终端入口,支持普通检索、GUI 高级检索、Excel/CSV 导出、引文及详情提取、停止和断点续抓。
|
|
45
|
+
|
|
46
|
+
## 安装和启动
|
|
47
|
+
|
|
48
|
+
Python 3.10 至 3.14,需要图形桌面和浏览器完成知网的人工验证。
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
python -m pip install "cnkibug[gui]"
|
|
52
|
+
python -m playwright install chromium
|
|
53
|
+
cnkibug-gui
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Linux 需要 Python Tk 支持。Debian/Ubuntu 可通过系统包管理器安装 `python3-tk`;建议在虚拟环境中安装 Python 依赖。Playwright 的系统依赖按其官方安装说明准备。
|
|
57
|
+
|
|
58
|
+
终端入口:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
python -m pip install cnkibug
|
|
62
|
+
python -m playwright install chromium
|
|
63
|
+
cnkibug
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Windows 用户可直接从 [GitHub Releases](https://github.com/KaffuAlcaid/CNKIBug/releases) 下载单文件 GUI EXE。
|
|
67
|
+
|
|
68
|
+
## 数据和更新
|
|
69
|
+
|
|
70
|
+
通过安装命令启动时,数据存放位置为:
|
|
71
|
+
|
|
72
|
+
- Linux:`$XDG_DATA_HOME/CNKIBug-data`,默认 `~/.local/share/CNKIBug-data`
|
|
73
|
+
- Windows:`%LOCALAPPDATA%/CNKIBug-data`
|
|
74
|
+
- macOS:`~/Library/Application Support/CNKIBug-data`,当前不在正式支持范围
|
|
75
|
+
|
|
76
|
+
导出目录由用户选择。通过源码入口或便携 EXE 启动时,仍使用启动文件旁的 `CNKIBug-data`。
|
|
77
|
+
|
|
78
|
+
Python 包通过 `python -m pip install --upgrade "cnkibug[gui]"` 升级;GUI 中的 EXE 自动替换仅适用于 Windows 打包版。
|
|
79
|
+
|
|
80
|
+
## 使用范围
|
|
81
|
+
|
|
82
|
+
项目与中国知网不存在隶属、授权或合作关系。验证码需要手动完成;不支持校园 WebVPN、统一认证网关或纯无头服务器。请遵守网站协议和所在机构规定,合理控制访问频率。
|
|
83
|
+
|
|
84
|
+
项目按 MIT License 提供,不保证网站长期兼容或结果完整。完整说明见 [项目仓库](https://github.com/KaffuAlcaid/CNKIBug)。
|
cnkibug-0.7.0/README.md
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
# CNKIBug
|
|
2
|
+
|
|
3
|
+
> 中国知网(CNKI)论文信息批量抓取工具。Windows 提供单文件 GUI `.exe`;Linux 可通过源码启动 GUI 或终端版本;macOS 可尝试通过源码运行终端版,当前不在正式支持范围。
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+

|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 功能
|
|
13
|
+
|
|
14
|
+
- GUI 和终端版都支持单个或批量检索
|
|
15
|
+
- GUI 提供高级检索(实验性),支持多组条件与普通检索混合执行,可在弹窗中编辑条件
|
|
16
|
+
- TXT 导入自动去重,开始前会显示任务规模和预计耗时
|
|
17
|
+
- 可导出 Excel 或 CSV,也可选抓取引文、关键词和摘要
|
|
18
|
+
- 自动保存配置、浏览器会话、日志和任务报告
|
|
19
|
+
- 中途可安全停止,并从最近完成页继续
|
|
20
|
+
- GUI 可手动检查更新、选择下载加速线路并测试连接
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## 运行截图
|
|
25
|
+
|
|
26
|
+
<table style="border: none;">
|
|
27
|
+
<tr>
|
|
28
|
+
<td style="text-align: center; vertical-align: top; width: 50%;">
|
|
29
|
+
<img src="docs/1.png" alt="GUI 任务设置" style="width: 100%; aspect-ratio: 25 / 18; object-fit: contain; border: 1px solid #ddd; border-radius: 4px;"/>
|
|
30
|
+
<br /><sub><b>输入关键词与设置</b></sub>
|
|
31
|
+
</td>
|
|
32
|
+
<td style="text-align: center; vertical-align: top; width: 50%;">
|
|
33
|
+
<img src="docs/3.png" alt="GUI 抓取过程" style="width: 100%; aspect-ratio: 25 / 18; object-fit: contain; border: 1px solid #ddd; border-radius: 4px;"/>
|
|
34
|
+
<br /><sub><b>GUI 抓取过程</b></sub>
|
|
35
|
+
</td>
|
|
36
|
+
</tr>
|
|
37
|
+
</table>
|
|
38
|
+
|
|
39
|
+
完整截图和操作说明见 [使用说明](docs/usage.md)。
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
## 快速开始
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
### 方式一:Windows 直接运行(推荐)
|
|
49
|
+
|
|
50
|
+
1. 前往 [Releases](../../releases) 页面下载 `CNKIBug-GUI.exe`
|
|
51
|
+
2. 确保电脑已安装 **Microsoft Edge**(Win10/11 通常已预装)
|
|
52
|
+
3. 启动后输入检索项,按界面提示确认任务
|
|
53
|
+
4. 手动通过知网滑块验证
|
|
54
|
+
5. 抓取完成后,在任务中选择的保存目录查看 Excel 或 CSV;配置、日志和任务报告保存在启动文件旁的 `CNKIBug-data/`
|
|
55
|
+
|
|
56
|
+
### 方式二:源码运行(Linux / macOS 用户或开发者)
|
|
57
|
+
|
|
58
|
+
先取得完整源码并进入项目根目录:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
git clone https://github.com/KaffuAlcaid/CNKIBug.git
|
|
62
|
+
cd CNKIBug
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
也可下载 GitHub 提供的 Source code 压缩包,解压后进入项目根目录。
|
|
66
|
+
|
|
67
|
+
#### Linux 终端版
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install -e .
|
|
71
|
+
playwright install chromium
|
|
72
|
+
python run.py
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
#### Linux GUI 版
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install -e ".[gui]"
|
|
79
|
+
playwright install chromium
|
|
80
|
+
python run_gui.py
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
#### macOS 终端版(非正式支持)
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
pip install -e .
|
|
87
|
+
playwright install chromium
|
|
88
|
+
python run.py
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
macOS 当前未纳入正式测试和支持范围。Release 中单独提供的 `CNKIBug-<版本>-source-tui.tar.gz` 只包含终端版源码,不包含 `run_gui.py` 和 `cnkibug/gui/`。
|
|
92
|
+
|
|
93
|
+
浏览器启动并完成滑块验证后,程序会将 Excel 或 CSV 写入任务中选择的保存目录;配置、日志和任务报告保存在项目根目录的 `CNKIBug-data/`。
|
|
94
|
+
|
|
95
|
+
> 源码运行需要可用的图形桌面完成滑块验证,Linux 使用 X11 / Wayland;无法在纯无头服务器运行。
|
|
96
|
+
|
|
97
|
+
### 更多说明
|
|
98
|
+
|
|
99
|
+
- [使用说明](docs/usage.md):导入、输出、进度和完整截图
|
|
100
|
+
- [配置说明](docs/configuration.md):`config.json` 和常见调整
|
|
101
|
+
- [开发和打包](docs/development.md):手动打包、项目结构和测试
|
|
102
|
+
|
|
103
|
+
## 系统要求
|
|
104
|
+
|
|
105
|
+
| 平台 | GUI | 终端版 |
|
|
106
|
+
|---------------|--------------|--------------|
|
|
107
|
+
| Windows 10/11 | `.exe` 或源码运行 | 源码运行 |
|
|
108
|
+
| Linux | 源码运行 | 源码运行 |
|
|
109
|
+
| macOS | 不支持 | 源码运行(非正式支持) |
|
|
110
|
+
|
|
111
|
+
| 项目 | 要求 |
|
|
112
|
+
|--------|-----------------------------------------------------------------------------------------|
|
|
113
|
+
| 浏览器 | Windows:Microsoft Edge(已预装或手动安装);Linux / macOS:`playwright install chromium` 的 Chromium |
|
|
114
|
+
| Python | 3.10–3.14(仅源码运行需要) |
|
|
115
|
+
| 图形桌面 | 所有抓取方式均需要人工通过知网滑块验证,无法在纯无头服务器运行 |
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
## 不支持范围
|
|
120
|
+
|
|
121
|
+
CNKIBug 面向中国知网(CNKI)检索结果及公开详情页信息抓取,GUI 高级检索处于实验阶段。不支持 Web of Science / SCI 数据库,也不支持通过校园 WebVPN、统一认证网关或代管账号密码的方式抓取机构资源。
|
|
122
|
+
|
|
123
|
+
遇到这类访问环境时,请改用浏览器手动访问对应平台。
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
## 免责声明
|
|
128
|
+
|
|
129
|
+
CNKIBug 是独立开发的开源工具,与中国知网(CNKI)及其关联方不存在隶属、授权、合作或背书关系。
|
|
130
|
+
|
|
131
|
+
本软件仅提供自动化操作能力。使用者应确保其访问和使用行为符合所在国家和地区合法的适用的法律法规、CNKI 用户协议及所在机构的相关规定,并自行确认对相关内容访问和处理权限。
|
|
132
|
+
|
|
133
|
+
请合理控制任务规模和访问频率,本项目不以绕过任何网站的技术或商业限制为目的,不鼓励或支持任何违反服务协议或适用法律法规的使用方式。
|
|
134
|
+
|
|
135
|
+
本软件按“现状”提供,不保证 CNKI 页面长期兼容,也不保证结果完整、准确或持续可用。
|
|
136
|
+
|
|
137
|
+
因网络异常、网站变更、验证码、账号或 IP 限制、数据处理及使用本软件产生的风险,由使用者依法承担;作者在所在国家和地区合法的适用法律允许的范围内不承担相关责任。
|
|
138
|
+
|
|
139
|
+
本免责声明作为项目文档的一部分,与仓库中的 MIT License 共同适用于本项目;如两者存在冲突,以适用法律规定为准。
|
|
140
|
+
|
|
141
|
+
软件会在本地保存配置、日志、任务状态和浏览器会话信息。请妥善保管运行数据目录,并在分享日志或任务报告前检查其中是否包含敏感信息。
|
|
142
|
+
|
|
143
|
+
本软件不会主动修改、去除或规避内容版权标识,也不会授予用户访问任何受版权保护内容的权利。
|
|
144
|
+
|
|
145
|
+
用户应自行妥善保管账号凭据,不建议在非受信任环境运行本软件。
|
|
146
|
+
|
|
147
|
+
下载并使用本软件,视作您同意本免责声明和 MIT License。
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
## 致谢 / Contributors
|
|
151
|
+
|
|
152
|
+
<table style="border: none;">
|
|
153
|
+
<tr>
|
|
154
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
155
|
+
<a href="https://github.com/KaffuAlcaid">
|
|
156
|
+
<img src="https://github.com/KaffuAlcaid.png" width="80px" alt="KaffuAlcaid"/>
|
|
157
|
+
<br /><sub><b>Kaffu_Alcaid</b></sub>
|
|
158
|
+
</a><br />核心开发
|
|
159
|
+
</td>
|
|
160
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
161
|
+
<a href="https://github.com/Speechlessyc">
|
|
162
|
+
<img src="https://github.com/Speechlessyc.png" width="80px" alt="Speechlessyc"/>
|
|
163
|
+
<br /><sub><b>Speechlessyc</b></sub>
|
|
164
|
+
</a><br />图标设计 & 测试
|
|
165
|
+
</td>
|
|
166
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
167
|
+
<a href="https://github.com/cloudw233">
|
|
168
|
+
<img src="https://github.com/cloudw233.png" width="80px" alt="cloudw233"/>
|
|
169
|
+
<br /><sub><b>cloudw233</b></sub>
|
|
170
|
+
</a><br />自动化构建(CI/CD)
|
|
171
|
+
</td>
|
|
172
|
+
</tr>
|
|
173
|
+
<tr>
|
|
174
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
175
|
+
<a href="https://github.com/zirend666-prog">
|
|
176
|
+
<img src="https://github.com/zirend666-prog.png" width="80px" alt="zirend666-prog"/>
|
|
177
|
+
<br /><sub><b>zirend666-prog</b></sub>
|
|
178
|
+
</a><br />产品经理
|
|
179
|
+
</td>
|
|
180
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
181
|
+
<a href="https://github.com/LuisCotton">
|
|
182
|
+
<img src="https://github.com/LuisCotton.png" width="80px" alt="LuisCotton"/>
|
|
183
|
+
<br /><sub><b>LuisCotton</b></sub>
|
|
184
|
+
</a><br />特约吉祥物
|
|
185
|
+
</td>
|
|
186
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
187
|
+
<a href="https://github.com/clover1909">
|
|
188
|
+
<img src="https://github.com/clover1909.png" width="80px" alt="clover1909"/>
|
|
189
|
+
<br /><sub><b>clover1909</b></sub>
|
|
190
|
+
</a><br />可爱群友
|
|
191
|
+
</td>
|
|
192
|
+
</tr>
|
|
193
|
+
<tr>
|
|
194
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
195
|
+
<img src="./logo.png" width="80px" alt="Placeholder"/>
|
|
196
|
+
<br /><sub><b>虚位以待</b></sub>
|
|
197
|
+
<br />欢迎提交 PR
|
|
198
|
+
</td>
|
|
199
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
200
|
+
<img src="./logo.png" width="80px" alt="Placeholder"/>
|
|
201
|
+
<br /><sub><b>虚位以待</b></sub>
|
|
202
|
+
<br />欢迎提交 PR
|
|
203
|
+
</td>
|
|
204
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
205
|
+
<img src="./logo.png" width="80px" alt="Placeholder"/>
|
|
206
|
+
<br /><sub><b>虚位以待</b></sub>
|
|
207
|
+
<br />欢迎提交 PR
|
|
208
|
+
</td>
|
|
209
|
+
</tr>
|
|
210
|
+
<tr>
|
|
211
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
212
|
+
<a href="https://openai.com/codex/">
|
|
213
|
+
<img src="https://github.com/openai.png" width="80px" alt="ChatGPT / Codex"/>
|
|
214
|
+
<br /><sub><b>ChatGPT / Codex</b></sub>
|
|
215
|
+
</a><br />代码改进
|
|
216
|
+
</td>
|
|
217
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
218
|
+
<a href="https://claude.ai">
|
|
219
|
+
<img src="https://github.com/claude.png" width="80px" alt="Claude"/>
|
|
220
|
+
<br /><sub><b>Claude</b></sub>
|
|
221
|
+
</a><br /> 代码改进
|
|
222
|
+
</td>
|
|
223
|
+
<td style="text-align: center; vertical-align: top; width: 200px;">
|
|
224
|
+
<a href="https://gemini.google.com/">
|
|
225
|
+
<img src="https://github.com/google.png" width="80px" alt="Gemini"/>
|
|
226
|
+
<br />
|
|
227
|
+
<sub><b>Gemini</b></sub>
|
|
228
|
+
</a>
|
|
229
|
+
<br />
|
|
230
|
+
代码审查<br/>
|
|
231
|
+
</td>
|
|
232
|
+
</tr>
|
|
233
|
+
</table>
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Application bootstrap, configuration, and console UI."""
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from ..core.events import EventSink
|
|
8
|
+
from ..core.memory import MemorySampler, format_task_finished_memory
|
|
9
|
+
from ..core.runtime import RuntimePaths
|
|
10
|
+
from ..core.settings import ScraperSettings, get_scraper_settings
|
|
11
|
+
from ..workflow.runner import scrape_cnki
|
|
12
|
+
from ..workflow.state import (
|
|
13
|
+
delete_last_task,
|
|
14
|
+
describe_task,
|
|
15
|
+
get_last_task_path,
|
|
16
|
+
load_last_task,
|
|
17
|
+
)
|
|
18
|
+
from .console import clear_screen, safe_input
|
|
19
|
+
from .environment import check_env
|
|
20
|
+
from .errors import _popup_error
|
|
21
|
+
from .events import ConsoleEventSink
|
|
22
|
+
from .prompts import collect_task_request
|
|
23
|
+
from .runtime import RuntimeState, init_runtime
|
|
24
|
+
from .ui import _console
|
|
25
|
+
from ..core.version import APP_VERSION
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
logging.basicConfig(
|
|
29
|
+
level=logging.ERROR,
|
|
30
|
+
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
_logger = logging.getLogger("cnkibug.app")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main(program_dir: Path) -> None:
|
|
37
|
+
runtime_state = _initialize_runtime(program_dir)
|
|
38
|
+
settings = get_scraper_settings(runtime_state.config)
|
|
39
|
+
memory_sampler = MemorySampler()
|
|
40
|
+
events = ConsoleEventSink(memory_sampler=memory_sampler)
|
|
41
|
+
try:
|
|
42
|
+
clear_screen()
|
|
43
|
+
_print_banner(runtime_state)
|
|
44
|
+
check_env()
|
|
45
|
+
|
|
46
|
+
while True:
|
|
47
|
+
try:
|
|
48
|
+
resume_action = _handle_pending_task(
|
|
49
|
+
runtime_state.paths,
|
|
50
|
+
settings,
|
|
51
|
+
events,
|
|
52
|
+
memory_sampler,
|
|
53
|
+
)
|
|
54
|
+
if resume_action == "exit":
|
|
55
|
+
break
|
|
56
|
+
if resume_action == "again":
|
|
57
|
+
clear_screen()
|
|
58
|
+
continue
|
|
59
|
+
|
|
60
|
+
request = collect_task_request(
|
|
61
|
+
detail_txt_export=settings.detail_txt_export,
|
|
62
|
+
config_path=runtime_state.paths.config_path,
|
|
63
|
+
)
|
|
64
|
+
if request is None:
|
|
65
|
+
break
|
|
66
|
+
_logger.info(
|
|
67
|
+
"用户选择: save_mode=%s keyword_count=%d pages=%d "
|
|
68
|
+
"include_citation=%s include_details=%s detail_txt_export=%s",
|
|
69
|
+
request.save_mode,
|
|
70
|
+
len(request.keywords),
|
|
71
|
+
request.max_pages,
|
|
72
|
+
request.include_citation,
|
|
73
|
+
request.include_details,
|
|
74
|
+
request.detail_txt_export,
|
|
75
|
+
)
|
|
76
|
+
_run_task(
|
|
77
|
+
memory_sampler,
|
|
78
|
+
request.keywords,
|
|
79
|
+
request.max_pages,
|
|
80
|
+
request.save_mode,
|
|
81
|
+
include_citation=request.include_citation,
|
|
82
|
+
include_details=request.include_details,
|
|
83
|
+
detail_txt_export=request.detail_txt_export,
|
|
84
|
+
settings=settings,
|
|
85
|
+
paths=runtime_state.paths,
|
|
86
|
+
events=events,
|
|
87
|
+
)
|
|
88
|
+
_logger.info("本轮抓取完成")
|
|
89
|
+
if _ask_run_again("\n[*] 本轮抓取已完成!是否清屏并开始新一轮抓取?(y/n): "):
|
|
90
|
+
clear_screen()
|
|
91
|
+
continue
|
|
92
|
+
_console.print("\n[bold green]感谢使用 CNKIBug,再见![/bold green]")
|
|
93
|
+
break
|
|
94
|
+
except RuntimeError as error:
|
|
95
|
+
_logger.warning("运行时错误: %s", error)
|
|
96
|
+
print(f"\n[!] {error}")
|
|
97
|
+
if safe_input("是否返回主菜单重试?(y/n): ").strip().lower() == "y":
|
|
98
|
+
_logger.info("用户选择返回主菜单重试")
|
|
99
|
+
continue
|
|
100
|
+
break
|
|
101
|
+
except Exception as error:
|
|
102
|
+
_logger.exception("程序遇到未知错误")
|
|
103
|
+
print("\n" + "!" * 40)
|
|
104
|
+
print(f" 程序遇到未知错误: {error}")
|
|
105
|
+
print("!" * 40)
|
|
106
|
+
break
|
|
107
|
+
except KeyboardInterrupt:
|
|
108
|
+
_logger.warning("用户中断,程序退出")
|
|
109
|
+
print("\n[*] 用户中断,程序退出。")
|
|
110
|
+
finally:
|
|
111
|
+
_logger.info("程序退出")
|
|
112
|
+
try:
|
|
113
|
+
input("\n按 [回车键 Enter] 退出程序...")
|
|
114
|
+
except (EOFError, KeyboardInterrupt):
|
|
115
|
+
pass
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _initialize_runtime(program_dir: Path) -> RuntimeState:
|
|
119
|
+
try:
|
|
120
|
+
return init_runtime(program_dir=program_dir, app_version=APP_VERSION)
|
|
121
|
+
except OSError as error:
|
|
122
|
+
if sys.platform == "win32":
|
|
123
|
+
_popup_error([
|
|
124
|
+
"==============================================",
|
|
125
|
+
" [致命错误] 无法创建运行数据目录!",
|
|
126
|
+
"----------------------------------------------",
|
|
127
|
+
f" 错误信息: {error}",
|
|
128
|
+
"",
|
|
129
|
+
" 请检查 run.py 或 CNKIBug.exe 所在目录的写入权限。",
|
|
130
|
+
"==============================================",
|
|
131
|
+
])
|
|
132
|
+
else:
|
|
133
|
+
print(f"[FATAL] 无法创建运行数据目录: {error}")
|
|
134
|
+
raise SystemExit(1) from error
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _print_banner(runtime_state: RuntimeState) -> None:
|
|
138
|
+
_console.print("=" * 50)
|
|
139
|
+
_console.print(" CNKI_Bug_dev | copyright by Kaffu_Alcaid")
|
|
140
|
+
_console.print(f" Version {APP_VERSION}")
|
|
141
|
+
_console.print("=" * 50)
|
|
142
|
+
_console.print(" 本软件用于抓取中国知网的论文标题\n")
|
|
143
|
+
_console.print("按 Ctrl+C 可随时中断并保存已抓取数据")
|
|
144
|
+
_console.print("それは,幾千の夜を舞う、さくらと少女たちの物語ーーー")
|
|
145
|
+
_console.print("祈祷着今后的你的人生,永远都有幸福的“魔法”相伴")
|
|
146
|
+
if any(level in {"WARNING", "ERROR"} for level, _ in runtime_state.events):
|
|
147
|
+
_console.print(
|
|
148
|
+
f"[yellow][配置提示] 配置文件或运行目录已自动调整,"
|
|
149
|
+
f"详情见日志:{runtime_state.log_path}[/yellow]"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _handle_pending_task(
|
|
154
|
+
paths: RuntimePaths,
|
|
155
|
+
settings: ScraperSettings,
|
|
156
|
+
events: EventSink,
|
|
157
|
+
memory_sampler: MemorySampler,
|
|
158
|
+
) -> str:
|
|
159
|
+
resume_state = load_last_task(paths)
|
|
160
|
+
last_task_path = get_last_task_path(paths)
|
|
161
|
+
if resume_state is None:
|
|
162
|
+
if last_task_path.exists():
|
|
163
|
+
if delete_last_task(paths) or not last_task_path.exists():
|
|
164
|
+
_console.print("[yellow][!] 检测到损坏的未完成任务缓存,已删除。[/yellow]")
|
|
165
|
+
else:
|
|
166
|
+
_console.print(
|
|
167
|
+
"[red][x] 损坏的断点文件无法删除,请手动删除后重新启动:"
|
|
168
|
+
f"{last_task_path}[/red]"
|
|
169
|
+
)
|
|
170
|
+
return "exit"
|
|
171
|
+
return "new"
|
|
172
|
+
|
|
173
|
+
_console.print("\n[yellow][!] 检测到上次未完成的抓取任务。[/yellow]")
|
|
174
|
+
_console.print(f" {describe_task(resume_state)}")
|
|
175
|
+
print(" 1 -> 继续上次任务")
|
|
176
|
+
print(" 2 -> 忽略断点,删除断点文件并开始新任务")
|
|
177
|
+
print(" 0 -> 保留断点并退出程序")
|
|
178
|
+
while True:
|
|
179
|
+
choice = safe_input("请输入选项(1、2 或 0): ").strip()
|
|
180
|
+
if choice not in {"0", "1", "2"}:
|
|
181
|
+
print("[!] 无效选项,请重新输入。")
|
|
182
|
+
continue
|
|
183
|
+
if choice == "0":
|
|
184
|
+
_logger.info("用户选择保留断点并退出")
|
|
185
|
+
return "exit"
|
|
186
|
+
if choice == "1":
|
|
187
|
+
break
|
|
188
|
+
if delete_last_task(paths) or not last_task_path.exists():
|
|
189
|
+
_logger.info("用户选择忽略并删除未完成任务断点")
|
|
190
|
+
_console.print("[yellow][*] 已忽略并删除上次任务断点。[/yellow]")
|
|
191
|
+
return "new"
|
|
192
|
+
_console.print(
|
|
193
|
+
"[red][x] 断点文件删除失败,尚未忽略该任务。"
|
|
194
|
+
f"请关闭占用文件的程序后重试:{last_task_path}[/red]"
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
_logger.info("用户选择继续未完成任务")
|
|
198
|
+
_run_task(
|
|
199
|
+
memory_sampler,
|
|
200
|
+
list(resume_state["keywords"]),
|
|
201
|
+
int(resume_state["max_pages"]),
|
|
202
|
+
str(resume_state["save_mode"]),
|
|
203
|
+
resume_state=resume_state,
|
|
204
|
+
settings=settings,
|
|
205
|
+
paths=paths,
|
|
206
|
+
events=events,
|
|
207
|
+
)
|
|
208
|
+
_logger.info("恢复任务结束")
|
|
209
|
+
if _ask_run_again("\n[*] 本轮抓取已结束!是否清屏并开始新一轮抓取?(y/n): "):
|
|
210
|
+
_logger.info("用户选择开始新一轮抓取")
|
|
211
|
+
return "again"
|
|
212
|
+
_console.print("\n[bold green]感谢使用 CNKIBug,再见![/bold green]")
|
|
213
|
+
return "exit"
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _run_task(
|
|
217
|
+
memory_sampler: MemorySampler,
|
|
218
|
+
keywords: list[str],
|
|
219
|
+
max_pages: int,
|
|
220
|
+
save_mode: str,
|
|
221
|
+
**kwargs,
|
|
222
|
+
) -> None:
|
|
223
|
+
memory_sampler.reset()
|
|
224
|
+
try:
|
|
225
|
+
scrape_cnki(keywords, max_pages, save_mode, **kwargs)
|
|
226
|
+
finally:
|
|
227
|
+
_console.print(format_task_finished_memory(memory_sampler.sample(force=True)))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _ask_run_again(prompt: str) -> bool:
|
|
231
|
+
return safe_input(prompt).strip().lower() == "y"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def clear_screen() -> None:
|
|
8
|
+
try:
|
|
9
|
+
if not sys.stdout.isatty():
|
|
10
|
+
return
|
|
11
|
+
except Exception:
|
|
12
|
+
return
|
|
13
|
+
if sys.platform == "win32":
|
|
14
|
+
os.system("cls") # noqa: S605, S607
|
|
15
|
+
sys.stdout.write("\033[3J")
|
|
16
|
+
sys.stdout.flush()
|
|
17
|
+
else:
|
|
18
|
+
sys.stdout.write("\033c")
|
|
19
|
+
sys.stdout.flush()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def safe_input(prompt: str = "") -> str:
|
|
23
|
+
try:
|
|
24
|
+
return input(prompt)
|
|
25
|
+
except EOFError:
|
|
26
|
+
print("\n[*] 检测到输入流结束(EOF),程序退出。")
|
|
27
|
+
raise SystemExit(0)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
|
|
2
|
+
# noinspection PyDeprecation
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
|
|
8
|
+
from .ui import _console
|
|
9
|
+
|
|
10
|
+
_EDGE_PATHS = [
|
|
11
|
+
r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
|
|
12
|
+
r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
|
|
13
|
+
os.path.expandvars(r"%LOCALAPPDATA%\Microsoft\Edge\Application\msedge.exe"),
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _edge_installed() -> bool:
|
|
18
|
+
if any(os.path.isfile(p) for p in _EDGE_PATHS):
|
|
19
|
+
return True
|
|
20
|
+
return shutil.which("msedge") is not None # noqa
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def check_env():
|
|
24
|
+
if sys.platform != "win32":
|
|
25
|
+
_home = os.path.expanduser("~")
|
|
26
|
+
if sys.platform == "darwin":
|
|
27
|
+
playwright_path = os.path.join(_home, "Library", "Caches", "ms-playwright")
|
|
28
|
+
else:
|
|
29
|
+
playwright_path = os.path.join(_home, ".cache", "ms-playwright")
|
|
30
|
+
_override = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
|
|
31
|
+
if _override and _override != "0":
|
|
32
|
+
playwright_path = _override
|
|
33
|
+
if not os.path.exists(playwright_path):
|
|
34
|
+
_console.print("\n[yellow][环境缺失] 请先在终端运行: playwright install chromium[/yellow]\n")
|
|
35
|
+
# 缺少必需依赖属于失败退出,用非 0 退出码,便于脚本化/CI 正确判断。
|
|
36
|
+
sys.exit(1)
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
if not _edge_installed():
|
|
40
|
+
_console.print(
|
|
41
|
+
"\n[yellow][环境提示] 未检测到 Microsoft Edge,"
|
|
42
|
+
"将尝试使用 Playwright Chromium。[/yellow]"
|
|
43
|
+
)
|
|
44
|
+
_console.print(
|
|
45
|
+
"[dim]若后续浏览器启动失败,请安装 Edge 或运行:"
|
|
46
|
+
"playwright install chromium[/dim]\n"
|
|
47
|
+
)
|