ktoolbox 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ktoolbox-0.3.0/PKG-INFO +191 -0
- ktoolbox-0.3.0/README.md +158 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/__init__.py +1 -1
- ktoolbox-0.3.0/ktoolbox/__main__.py +20 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/__init__.py +1 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/job.py +4 -8
- ktoolbox-0.3.0/ktoolbox/action/utils.py +14 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/base.py +19 -3
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/cli.py +2 -3
- ktoolbox-0.3.0/ktoolbox/configuration.py +157 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/downloader.py +26 -5
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/utils.py +45 -11
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/pyproject.toml +19 -9
- ktoolbox-0.2.2/PKG-INFO +0 -272
- ktoolbox-0.2.2/README.md +0 -239
- ktoolbox-0.2.2/ktoolbox/__main__.py +0 -13
- ktoolbox-0.2.2/ktoolbox/configuration.py +0 -134
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/LICENSE +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/base.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/fetch.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/search.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/misc/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/misc/get_app_version.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/announcement.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/creator.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/post.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_announcement.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_creator_post.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_creators.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_post.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/utils.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/base.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/enum.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/__init__.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/model.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/runner.py +0 -0
- {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/model.py +0 -0
ktoolbox-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: ktoolbox
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: A useful CLI tool for downloading posts in Kemono.party / .su
|
|
5
|
+
Home-page: https://ktoolbox.readthedocs.io/
|
|
6
|
+
Keywords: kemono,kemono.party,cli-app,downloader,os-independent
|
|
7
|
+
Author: Ljzd-PRO
|
|
8
|
+
Author-email: ljzd@office.ljzd-pro.asia
|
|
9
|
+
Requires-Python: >=3.8,<3.12
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Topic :: Terminals
|
|
19
|
+
Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
|
|
20
|
+
Requires-Dist: fire (>=0.5.0,<0.6.0)
|
|
21
|
+
Requires-Dist: httpx[socks] (>=0.25.2,<0.26.0)
|
|
22
|
+
Requires-Dist: loguru (>=0.7.2,<0.8.0)
|
|
23
|
+
Requires-Dist: pathvalidate (>=3.2.0,<4.0.0)
|
|
24
|
+
Requires-Dist: pydantic (>=2.4.2,<3.0.0)
|
|
25
|
+
Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
|
|
26
|
+
Requires-Dist: tenacity (>=8.2.3,<9.0.0)
|
|
27
|
+
Requires-Dist: tqdm (>=4.66.1,<5.0.0)
|
|
28
|
+
Project-URL: Bug Tracker, https://github.com/Ljzd-PRO/KToolBox/issues
|
|
29
|
+
Project-URL: Documentation, https://ktoolbox.readthedocs.io/
|
|
30
|
+
Project-URL: Repository, https://github.com/Ljzd-PRO/KToolBox
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
<p align="center" style="text-decoration:none">
|
|
34
|
+
<img align="center" src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/repository-open-graph-2.svg" alt="logo">
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
<h1 align="center">
|
|
38
|
+
KToolBox
|
|
39
|
+
</h1>
|
|
40
|
+
|
|
41
|
+
<p align="center">
|
|
42
|
+
KToolBox is a useful CLI tool for downloading posts content in
|
|
43
|
+
<a href="https://kemono.su/">Kemono.party / Kemono.su</a>
|
|
44
|
+
</p>
|
|
45
|
+
|
|
46
|
+
<p align="center">
|
|
47
|
+
<a href="https://pypi.org/project/ktoolbox" target="_blank">
|
|
48
|
+
<img src="https://img.shields.io/github/v/release/Ljzd-PRO/KToolBox?logo=python" alt="Version">
|
|
49
|
+
</a>
|
|
50
|
+
|
|
51
|
+
<a href="./LICENSE">
|
|
52
|
+
<img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
|
|
53
|
+
</a>
|
|
54
|
+
|
|
55
|
+
<a href="https://github.com/Ljzd-PRO/KToolBox/activity">
|
|
56
|
+
<img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
|
|
57
|
+
</a>
|
|
58
|
+
|
|
59
|
+
<a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
|
|
60
|
+
<img src="https://codecov.io/gh/Ljzd-PRO/KToolBox/branch/master/graph/badge.svg?token=5XK9CYQHQN" alt="codecov"/>
|
|
61
|
+
</a>
|
|
62
|
+
|
|
63
|
+
<a href='https://ktoolbox.readthedocs.io/'>
|
|
64
|
+
<img src='https://readthedocs.org/projects/ktoolbox/badge/?version=latest' alt='Documentation Status' />
|
|
65
|
+
</a>
|
|
66
|
+
|
|
67
|
+
<a style="text-decoration:none">
|
|
68
|
+
<img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
|
|
69
|
+
</a>
|
|
70
|
+
</p>
|
|
71
|
+
|
|
72
|
+
<p align="center">
|
|
73
|
+
<a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
|
|
74
|
+
</p>
|
|
75
|
+
|
|
76
|
+
<img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
|
|
77
|
+
|
|
78
|
+
## Features
|
|
79
|
+
|
|
80
|
+
- Support for **multi-thread** downloads (technically, coroutine)
|
|
81
|
+
- **Retry** after download failed
|
|
82
|
+
- Ability to download individual post as well as **all post** by a specified creator/artist
|
|
83
|
+
- **Update downloaded** creator/artist directories to the latest status
|
|
84
|
+
- Customize the **structure** of downloaded post/creator **directories**
|
|
85
|
+
- Search for creators/artists and posts, and **export the results**
|
|
86
|
+
- **Cross-platform** support & **iOS shortcuts** available
|
|
87
|
+
|
|
88
|
+
## Dev Plan
|
|
89
|
+
|
|
90
|
+
- [ ] Add a Fluent Design style GUI
|
|
91
|
+
- [x] Add uvloop support for Unix platform
|
|
92
|
+
|
|
93
|
+
## Tutorial
|
|
94
|
+
|
|
95
|
+
See [documentation](https://ktoolbox.readthedocs.io/) for more details.
|
|
96
|
+
|
|
97
|
+
### Installation
|
|
98
|
+
|
|
99
|
+
- Normal
|
|
100
|
+
```bash
|
|
101
|
+
pip3 install ktoolbox
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
- For [a-Shell](https://github.com/holzschu/a-shell)
|
|
105
|
+
```bash
|
|
106
|
+
pip3 install ktoolbox-pure-py
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Command
|
|
110
|
+
|
|
111
|
+
For more information, use the help command or goto [Command](https://ktoolbox.readthedocs.io/latest/commands/) page.
|
|
112
|
+
|
|
113
|
+
#### ❓ Get general help
|
|
114
|
+
```bash
|
|
115
|
+
ktoolbox -h
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
#### ❓ Get help of a command
|
|
119
|
+
```bash
|
|
120
|
+
ktoolbox download-post -h
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
#### ⬇️🖼️ Download a specific post
|
|
124
|
+
```bash
|
|
125
|
+
ktoolbox download-post https://kemono.su/fanbox/user/49494721/post/6608808
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
If some files failed to download, you can try to execute the command line again,
|
|
129
|
+
the downloaded files will be **skipped**.
|
|
130
|
+
|
|
131
|
+
#### ⬇️🖌️ Download all posts from a creator
|
|
132
|
+
```bash
|
|
133
|
+
ktoolbox sync-creator https://kemono.su/fanbox/user/9016
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
> By default, you will get a `creator-indices.ktoolbox` file in the creator directory,
|
|
137
|
+
> you can use it to update the directory anytime.
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
#### 🔄️ Update a downloaded creator directory
|
|
141
|
+
```bash
|
|
142
|
+
ktoolbox sync-creator https://kemono.su/fanbox/user/641955 --update-with=./xxx/creator-indices.ktoolbox
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.
|
|
146
|
+
|
|
147
|
+
### iOS Shortcuts
|
|
148
|
+
|
|
149
|
+
Goto [Shortcuts for iOS](https://ktoolbox.readthedocs.io/latest/shortcut/) page for more details.
|
|
150
|
+
|
|
151
|
+
### Configuration
|
|
152
|
+
|
|
153
|
+
Goto [Configuration-Guide](https://ktoolbox.readthedocs.io/latest/configuration/guide/) page for more details.
|
|
154
|
+
|
|
155
|
+
## Other Branches
|
|
156
|
+
|
|
157
|
+
- Pure Python branch: [🔗pure-py](https://github.com/Ljzd-PRO/KToolBox/tree/pure-py)
|
|
158
|
+
- Use pydantic v1 so that cargo is not needed for installation
|
|
159
|
+
- For example, you can use it on iOS terminal App [a-Shell](https://github.com/holzschu/a-shell)
|
|
160
|
+
- 🔗[PyPI](https://pypi.org/project/ktoolbox-pure-py/)
|
|
161
|
+
- Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
|
|
162
|
+
|
|
163
|
+
## About Kemono
|
|
164
|
+
|
|
165
|
+
Description from https://kemono.su :
|
|
166
|
+
|
|
167
|
+
> Kemono is a public archiver for:
|
|
168
|
+
>
|
|
169
|
+
> - Patreon
|
|
170
|
+
> - Pixiv Fanbox
|
|
171
|
+
> - Discord
|
|
172
|
+
> - Fantia
|
|
173
|
+
> - Afdian
|
|
174
|
+
> - Boosty
|
|
175
|
+
> - DLsite
|
|
176
|
+
> - Gumroad
|
|
177
|
+
> - SubscribeStar
|
|
178
|
+
>
|
|
179
|
+
> Contributors here upload content and share it here for easy searching and organization. \
|
|
180
|
+
> To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
|
|
181
|
+
|
|
182
|
+
## Code Coverage
|
|
183
|
+
|
|
184
|
+

|
|
185
|
+
|
|
186
|
+
## License
|
|
187
|
+
|
|
188
|
+
KToolBox is licensed under MIT.
|
|
189
|
+
|
|
190
|
+
Copyright © 2023 by Ljzd-PRO.
|
|
191
|
+
|
ktoolbox-0.3.0/README.md
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
<p align="center" style="text-decoration:none">
|
|
2
|
+
<img align="center" src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/repository-open-graph-2.svg" alt="logo">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">
|
|
6
|
+
KToolBox
|
|
7
|
+
</h1>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
KToolBox is a useful CLI tool for downloading posts content in
|
|
11
|
+
<a href="https://kemono.su/">Kemono.party / Kemono.su</a>
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<p align="center">
|
|
15
|
+
<a href="https://pypi.org/project/ktoolbox" target="_blank">
|
|
16
|
+
<img src="https://img.shields.io/github/v/release/Ljzd-PRO/KToolBox?logo=python" alt="Version">
|
|
17
|
+
</a>
|
|
18
|
+
|
|
19
|
+
<a href="./LICENSE">
|
|
20
|
+
<img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
|
|
21
|
+
</a>
|
|
22
|
+
|
|
23
|
+
<a href="https://github.com/Ljzd-PRO/KToolBox/activity">
|
|
24
|
+
<img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
|
|
25
|
+
</a>
|
|
26
|
+
|
|
27
|
+
<a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
|
|
28
|
+
<img src="https://codecov.io/gh/Ljzd-PRO/KToolBox/branch/master/graph/badge.svg?token=5XK9CYQHQN" alt="codecov"/>
|
|
29
|
+
</a>
|
|
30
|
+
|
|
31
|
+
<a href='https://ktoolbox.readthedocs.io/'>
|
|
32
|
+
<img src='https://readthedocs.org/projects/ktoolbox/badge/?version=latest' alt='Documentation Status' />
|
|
33
|
+
</a>
|
|
34
|
+
|
|
35
|
+
<a style="text-decoration:none">
|
|
36
|
+
<img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
|
|
37
|
+
</a>
|
|
38
|
+
</p>
|
|
39
|
+
|
|
40
|
+
<p align="center">
|
|
41
|
+
<a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
|
|
42
|
+
</p>
|
|
43
|
+
|
|
44
|
+
<img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
|
|
45
|
+
|
|
46
|
+
## Features
|
|
47
|
+
|
|
48
|
+
- Support for **multi-thread** downloads (technically, coroutine)
|
|
49
|
+
- **Retry** after download failed
|
|
50
|
+
- Ability to download individual post as well as **all post** by a specified creator/artist
|
|
51
|
+
- **Update downloaded** creator/artist directories to the latest status
|
|
52
|
+
- Customize the **structure** of downloaded post/creator **directories**
|
|
53
|
+
- Search for creators/artists and posts, and **export the results**
|
|
54
|
+
- **Cross-platform** support & **iOS shortcuts** available
|
|
55
|
+
|
|
56
|
+
## Dev Plan
|
|
57
|
+
|
|
58
|
+
- [ ] Add a Fluent Design style GUI
|
|
59
|
+
- [x] Add uvloop support for Unix platform
|
|
60
|
+
|
|
61
|
+
## Tutorial
|
|
62
|
+
|
|
63
|
+
See [documentation](https://ktoolbox.readthedocs.io/) for more details.
|
|
64
|
+
|
|
65
|
+
### Installation
|
|
66
|
+
|
|
67
|
+
- Normal
|
|
68
|
+
```bash
|
|
69
|
+
pip3 install ktoolbox
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
- For [a-Shell](https://github.com/holzschu/a-shell)
|
|
73
|
+
```bash
|
|
74
|
+
pip3 install ktoolbox-pure-py
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Command
|
|
78
|
+
|
|
79
|
+
For more information, use the help command or goto [Command](https://ktoolbox.readthedocs.io/latest/commands/) page.
|
|
80
|
+
|
|
81
|
+
#### ❓ Get general help
|
|
82
|
+
```bash
|
|
83
|
+
ktoolbox -h
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
#### ❓ Get help of a command
|
|
87
|
+
```bash
|
|
88
|
+
ktoolbox download-post -h
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
#### ⬇️🖼️ Download a specific post
|
|
92
|
+
```bash
|
|
93
|
+
ktoolbox download-post https://kemono.su/fanbox/user/49494721/post/6608808
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
If some files failed to download, you can try to execute the command line again,
|
|
97
|
+
the downloaded files will be **skipped**.
|
|
98
|
+
|
|
99
|
+
#### ⬇️🖌️ Download all posts from a creator
|
|
100
|
+
```bash
|
|
101
|
+
ktoolbox sync-creator https://kemono.su/fanbox/user/9016
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
> By default, you will get a `creator-indices.ktoolbox` file in the creator directory,
|
|
105
|
+
> you can use it to update the directory anytime.
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
#### 🔄️ Update a downloaded creator directory
|
|
109
|
+
```bash
|
|
110
|
+
ktoolbox sync-creator https://kemono.su/fanbox/user/641955 --update-with=./xxx/creator-indices.ktoolbox
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.
|
|
114
|
+
|
|
115
|
+
### iOS Shortcuts
|
|
116
|
+
|
|
117
|
+
Goto [Shortcuts for iOS](https://ktoolbox.readthedocs.io/latest/shortcut/) page for more details.
|
|
118
|
+
|
|
119
|
+
### Configuration
|
|
120
|
+
|
|
121
|
+
Goto [Configuration-Guide](https://ktoolbox.readthedocs.io/latest/configuration/guide/) page for more details.
|
|
122
|
+
|
|
123
|
+
## Other Branches
|
|
124
|
+
|
|
125
|
+
- Pure Python branch: [🔗pure-py](https://github.com/Ljzd-PRO/KToolBox/tree/pure-py)
|
|
126
|
+
- Use pydantic v1 so that cargo is not needed for installation
|
|
127
|
+
- For example, you can use it on iOS terminal App [a-Shell](https://github.com/holzschu/a-shell)
|
|
128
|
+
- 🔗[PyPI](https://pypi.org/project/ktoolbox-pure-py/)
|
|
129
|
+
- Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
|
|
130
|
+
|
|
131
|
+
## About Kemono
|
|
132
|
+
|
|
133
|
+
Description from https://kemono.su :
|
|
134
|
+
|
|
135
|
+
> Kemono is a public archiver for:
|
|
136
|
+
>
|
|
137
|
+
> - Patreon
|
|
138
|
+
> - Pixiv Fanbox
|
|
139
|
+
> - Discord
|
|
140
|
+
> - Fantia
|
|
141
|
+
> - Afdian
|
|
142
|
+
> - Boosty
|
|
143
|
+
> - DLsite
|
|
144
|
+
> - Gumroad
|
|
145
|
+
> - SubscribeStar
|
|
146
|
+
>
|
|
147
|
+
> Contributors here upload content and share it here for easy searching and organization. \
|
|
148
|
+
> To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
|
|
149
|
+
|
|
150
|
+
## Code Coverage
|
|
151
|
+
|
|
152
|
+

|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
KToolBox is licensed under MIT.
|
|
157
|
+
|
|
158
|
+
Copyright © 2023 by Ljzd-PRO.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import fire
|
|
2
|
+
from loguru import logger
|
|
3
|
+
|
|
4
|
+
from ktoolbox.cli import KToolBoxCli
|
|
5
|
+
from ktoolbox.configuration import config
|
|
6
|
+
from ktoolbox.utils import logger_init, uvloop_init, generate_msg
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main():
|
|
10
|
+
try:
|
|
11
|
+
logger_init(cli_use=True)
|
|
12
|
+
logger.debug(generate_msg(config=config))
|
|
13
|
+
uvloop_init()
|
|
14
|
+
fire.Fire(KToolBoxCli)
|
|
15
|
+
except KeyboardInterrupt:
|
|
16
|
+
logger.error("KToolBox was interrupted by the user")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
if __name__ == "__main__":
|
|
20
|
+
main()
|
|
@@ -6,6 +6,7 @@ from loguru import logger
|
|
|
6
6
|
from pathvalidate import sanitize_filename
|
|
7
7
|
|
|
8
8
|
from ktoolbox.action import ActionRet, fetch_all_creator_posts, FetchInterruptError
|
|
9
|
+
from ktoolbox.action.utils import generate_post_path_name
|
|
9
10
|
from ktoolbox.api.model import Post
|
|
10
11
|
from ktoolbox.api.posts import get_creator_post
|
|
11
12
|
from ktoolbox.configuration import config, PostStructureConfiguration
|
|
@@ -47,7 +48,7 @@ async def create_job_from_post(
|
|
|
47
48
|
|
|
48
49
|
# Create jobs
|
|
49
50
|
jobs: List[Job] = []
|
|
50
|
-
for attachment in post.attachments: # attachments
|
|
51
|
+
for attachment in post.attachments or []: # attachments
|
|
51
52
|
if not attachment.path:
|
|
52
53
|
continue
|
|
53
54
|
jobs.append(
|
|
@@ -58,7 +59,7 @@ async def create_job_from_post(
|
|
|
58
59
|
type=PostFileTypeEnum.Attachment
|
|
59
60
|
)
|
|
60
61
|
)
|
|
61
|
-
if post.file.path: # file
|
|
62
|
+
if post.file and post.file.path: # file
|
|
62
63
|
jobs.append(
|
|
63
64
|
Job(
|
|
64
65
|
path=post_path,
|
|
@@ -170,12 +171,7 @@ async def create_job_from_creator(
|
|
|
170
171
|
job_list: List[Job] = []
|
|
171
172
|
for post in post_list:
|
|
172
173
|
# Get post path
|
|
173
|
-
if mix_posts
|
|
174
|
-
default_post_path = path
|
|
175
|
-
elif config.job.post_id_as_path:
|
|
176
|
-
default_post_path = path / post.id
|
|
177
|
-
else:
|
|
178
|
-
default_post_path = path / sanitize_filename(post.title)
|
|
174
|
+
default_post_path = path if mix_posts else path / generate_post_path_name(post)
|
|
179
175
|
if update_from:
|
|
180
176
|
if not (post_path := update_from.posts_path.get(post.id)):
|
|
181
177
|
post_path = default_post_path
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from pathvalidate import sanitize_filename
|
|
2
|
+
|
|
3
|
+
from ktoolbox.api.model import Post
|
|
4
|
+
from ktoolbox.configuration import config
|
|
5
|
+
|
|
6
|
+
__all__ = ["generate_post_path_name"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def generate_post_path_name(post: Post) -> str:
|
|
10
|
+
"""Generate directory name for post to save."""
|
|
11
|
+
if config.job.post_id_as_path or not post.title:
|
|
12
|
+
return post.id
|
|
13
|
+
else:
|
|
14
|
+
return sanitize_filename(post.title)
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
-
from typing import Literal, Generic, TypeVar, Optional, Callable
|
|
2
|
+
from typing import Literal, Generic, TypeVar, Optional, Callable
|
|
3
3
|
from urllib.parse import urlunparse
|
|
4
4
|
|
|
5
5
|
import httpx
|
|
6
6
|
import tenacity
|
|
7
|
+
from loguru import logger
|
|
7
8
|
from pydantic import BaseModel, ValidationError, RootModel
|
|
8
9
|
from tenacity import RetryCallState, wait_fixed, retry_if_result
|
|
9
10
|
from tenacity.stop import stop_base, stop_never, stop_after_attempt
|
|
10
11
|
|
|
11
12
|
from ktoolbox.configuration import config
|
|
12
13
|
from ktoolbox.enum import RetCodeEnum
|
|
13
|
-
from ktoolbox.utils import BaseRet
|
|
14
|
+
from ktoolbox.utils import BaseRet, generate_msg
|
|
14
15
|
|
|
15
16
|
__all__ = ["APITenacityStop", "APIRet", "BaseAPI"]
|
|
16
17
|
|
|
@@ -27,6 +28,21 @@ class APITenacityStop(stop_base):
|
|
|
27
28
|
return stop_after_attempt(config.api.retry_times)(retry_state)
|
|
28
29
|
|
|
29
30
|
|
|
31
|
+
def _retry_error_callback(state: RetryCallState) -> "APIRet":
|
|
32
|
+
"""
|
|
33
|
+
Call after all reties failed
|
|
34
|
+
:return Keep the origin return value
|
|
35
|
+
"""
|
|
36
|
+
# noinspection SpellCheckingInspection
|
|
37
|
+
logger.error(
|
|
38
|
+
generate_msg(
|
|
39
|
+
f"Kemono API call failed",
|
|
40
|
+
ret=state.outcome.result(),
|
|
41
|
+
)
|
|
42
|
+
)
|
|
43
|
+
return state.outcome.result()
|
|
44
|
+
|
|
45
|
+
|
|
30
46
|
def _retry(*args, **kwargs):
|
|
31
47
|
"""Wrap an API method with a new ``Retrying`` object"""
|
|
32
48
|
wrapper = tenacity.retry(
|
|
@@ -34,7 +50,7 @@ def _retry(*args, **kwargs):
|
|
|
34
50
|
wait=wait_fixed(config.api.retry_interval),
|
|
35
51
|
retry=retry_if_result(lambda x: not bool(x)),
|
|
36
52
|
reraise=True,
|
|
37
|
-
retry_error_callback=
|
|
53
|
+
retry_error_callback=_retry_error_callback,
|
|
38
54
|
**kwargs
|
|
39
55
|
)
|
|
40
56
|
if len(args) == 1 and callable(args[0]):
|
|
@@ -6,7 +6,7 @@ from loguru import logger
|
|
|
6
6
|
from pathvalidate import sanitize_filename
|
|
7
7
|
|
|
8
8
|
from ktoolbox import __version__
|
|
9
|
-
from ktoolbox.action import create_job_from_post, create_job_from_creator
|
|
9
|
+
from ktoolbox.action import create_job_from_post, create_job_from_creator, generate_post_path_name
|
|
10
10
|
from ktoolbox.action import search_creator as search_creator_action, search_creator_post as search_creator_post_action
|
|
11
11
|
from ktoolbox.api.misc import get_app_version
|
|
12
12
|
from ktoolbox.api.posts import get_post as get_post_api
|
|
@@ -171,7 +171,7 @@ class KToolBoxCli:
|
|
|
171
171
|
post_id=post_id
|
|
172
172
|
)
|
|
173
173
|
if ret:
|
|
174
|
-
post_path = path /
|
|
174
|
+
post_path = path / generate_post_path_name(ret.data)
|
|
175
175
|
job_list = await create_job_from_post(
|
|
176
176
|
post=ret.data,
|
|
177
177
|
post_path=post_path,
|
|
@@ -224,7 +224,6 @@ class KToolBoxCli:
|
|
|
224
224
|
You can update the directory anytime after download finished, \
|
|
225
225
|
such as to update after creator published new posts.
|
|
226
226
|
|
|
227
|
-
.. rubric:: NOTICE:
|
|
228
227
|
* If ``update_from`` was provided, it should be located **inside the creator directory**.
|
|
229
228
|
|
|
230
229
|
:param url: The post URL
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Literal, Union, Optional
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel
|
|
7
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"config",
|
|
11
|
+
"APIConfiguration",
|
|
12
|
+
"DownloaderConfiguration",
|
|
13
|
+
"PostStructureConfiguration",
|
|
14
|
+
"JobConfiguration",
|
|
15
|
+
"Configuration"
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# noinspection SpellCheckingInspection,GrazieInspection
|
|
20
|
+
class APIConfiguration(BaseModel):
|
|
21
|
+
"""
|
|
22
|
+
Kemono API Configuration
|
|
23
|
+
|
|
24
|
+
:ivar scheme: Kemono API URL scheme
|
|
25
|
+
:ivar netloc: Kemono API URL netloc
|
|
26
|
+
:ivar statics_netloc: URL netloc of Kemono server for static files (e.g. images)
|
|
27
|
+
:ivar files_netloc: URL netloc of Kemono server for post files
|
|
28
|
+
:ivar path: Kemono API URL root path
|
|
29
|
+
:ivar timeout: API request timeout
|
|
30
|
+
:ivar retry_times: API request retry times (when request failed)
|
|
31
|
+
:ivar retry_interval: Seconds of API request retry interval
|
|
32
|
+
"""
|
|
33
|
+
scheme: Literal["http", "https"] = "https"
|
|
34
|
+
netloc: str = "kemono.su"
|
|
35
|
+
statics_netloc: str = "img.kemono.su"
|
|
36
|
+
files_netloc: str = "kemono.su"
|
|
37
|
+
path: str = "/api/v1"
|
|
38
|
+
timeout: float = 5.0
|
|
39
|
+
retry_times: int = 3
|
|
40
|
+
retry_interval: float = 2.0
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class DownloaderConfiguration(BaseModel):
|
|
44
|
+
"""
|
|
45
|
+
File Downloader Configuration
|
|
46
|
+
|
|
47
|
+
:ivar scheme: Downloader URL scheme
|
|
48
|
+
:ivar timeout: Downloader request timeout
|
|
49
|
+
:ivar encoding: Charset for filename parsing and post content text saving
|
|
50
|
+
:ivar buffer_size: Number of bytes of file I/O buffer for each downloading file
|
|
51
|
+
:ivar chunk_size: Number of bytes of chunk of downloader stream
|
|
52
|
+
:ivar temp_suffix: Temp filename suffix of downloading files
|
|
53
|
+
:ivar retry_times: Downloader retry times (when download failed)
|
|
54
|
+
:ivar retry_stop_never: Never stop downloader from retrying (when download failed) \
|
|
55
|
+
(``retry_times`` will be ignored when enabled)
|
|
56
|
+
:ivar retry_interval: Seconds of downloader retry interval
|
|
57
|
+
"""
|
|
58
|
+
scheme: Literal["http", "https"] = "https"
|
|
59
|
+
timeout: float = 30.0
|
|
60
|
+
encoding: str = "utf-8"
|
|
61
|
+
buffer_size: int = 20480
|
|
62
|
+
chunk_size: int = 1024
|
|
63
|
+
temp_suffix: str = "tmp"
|
|
64
|
+
retry_times: int = 10
|
|
65
|
+
retry_stop_never: bool = False
|
|
66
|
+
retry_interval: float = 3.0
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class PostStructureConfiguration(BaseModel):
|
|
70
|
+
# noinspection SpellCheckingInspection
|
|
71
|
+
"""
|
|
72
|
+
Post path structure model
|
|
73
|
+
|
|
74
|
+
- Default:
|
|
75
|
+
```
|
|
76
|
+
|-- ..
|
|
77
|
+
|-- attachments
|
|
78
|
+
| |-- 1.png
|
|
79
|
+
| |-- 2.png
|
|
80
|
+
|-- content.txt
|
|
81
|
+
|-- <Post file>
|
|
82
|
+
|-- <Post data (post.ktoolbox.json)>
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
:ivar attachments: Sub path of attachment directory
|
|
86
|
+
:ivar content_filepath: Sub path of post content HTML file
|
|
87
|
+
"""
|
|
88
|
+
attachments: Path = Path("attachments")
|
|
89
|
+
content_filepath: Path = Path("index.html")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class JobConfiguration(BaseModel):
|
|
93
|
+
"""
|
|
94
|
+
Download jobs Configuration
|
|
95
|
+
|
|
96
|
+
:ivar count: Number of coroutines for concurrent download
|
|
97
|
+
:ivar post_id_as_path: Use post ID as post directory name
|
|
98
|
+
:ivar post_structure: Post path structure
|
|
99
|
+
:ivar mix_posts: Save all files from different posts at same path in creator directory. \
|
|
100
|
+
It would not create any post directory, and ``CreatorIndices`` would not been recorded, \
|
|
101
|
+
without ``CreatorIndices`` you **cannot update** the creator directory.
|
|
102
|
+
:ivar job_list_filepath: Filepath for job list data saving, ``None`` for disable job list saving
|
|
103
|
+
"""
|
|
104
|
+
count: int = 4
|
|
105
|
+
post_id_as_path: bool = False
|
|
106
|
+
post_structure: PostStructureConfiguration = PostStructureConfiguration()
|
|
107
|
+
mix_posts: bool = False
|
|
108
|
+
job_list_filepath: Optional[Path] = None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class LoggerConfiguration(BaseModel):
|
|
112
|
+
"""
|
|
113
|
+
Logger configuration
|
|
114
|
+
|
|
115
|
+
:ivar path: Path to save logs, ``None`` for disable log file output
|
|
116
|
+
:ivar level: Log filter level
|
|
117
|
+
:ivar rotation: Log rotation
|
|
118
|
+
"""
|
|
119
|
+
path: Optional[Path] = Path("logs")
|
|
120
|
+
level: Union[str, int] = logging.DEBUG
|
|
121
|
+
rotation: Union[str, int, datetime.time, datetime.timedelta] = "1 week"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class Configuration(BaseSettings):
|
|
125
|
+
# noinspection SpellCheckingInspection,GrazieInspection
|
|
126
|
+
"""
|
|
127
|
+
KToolBox Configuration
|
|
128
|
+
|
|
129
|
+
:ivar api:
|
|
130
|
+
:ivar downloader:
|
|
131
|
+
:ivar job:
|
|
132
|
+
:ivar logger:
|
|
133
|
+
:ivar ssl_verify: Enable SSL certificate verification for Kemono API server and download server
|
|
134
|
+
:ivar json_dump_indent: Indent of JSON file dump
|
|
135
|
+
:ivar use_uvloop: Use uvloop for asyncio (Disabled on Windows by default) \
|
|
136
|
+
uvloop will improve concurrent performance, but it is not compatible with Windows. \
|
|
137
|
+
Install uvloop by `pip install ktoolbox[uvloop]` or it will not work.
|
|
138
|
+
"""
|
|
139
|
+
api: APIConfiguration = APIConfiguration()
|
|
140
|
+
downloader: DownloaderConfiguration = DownloaderConfiguration()
|
|
141
|
+
job: JobConfiguration = JobConfiguration()
|
|
142
|
+
logger: LoggerConfiguration = LoggerConfiguration()
|
|
143
|
+
|
|
144
|
+
ssl_verify: bool = True
|
|
145
|
+
json_dump_indent: int = 4
|
|
146
|
+
use_uvloop: bool = True
|
|
147
|
+
|
|
148
|
+
# noinspection SpellCheckingInspection
|
|
149
|
+
model_config = SettingsConfigDict(
|
|
150
|
+
env_prefix='ktoolbox_',
|
|
151
|
+
env_nested_delimiter='__',
|
|
152
|
+
env_file='.env',
|
|
153
|
+
env_file_encoding='utf-8'
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
config = Configuration(_env_file='prod.env')
|