ktoolbox 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. ktoolbox-0.3.0/PKG-INFO +191 -0
  2. ktoolbox-0.3.0/README.md +158 -0
  3. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/__init__.py +1 -1
  4. ktoolbox-0.3.0/ktoolbox/__main__.py +20 -0
  5. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/__init__.py +1 -0
  6. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/job.py +4 -8
  7. ktoolbox-0.3.0/ktoolbox/action/utils.py +14 -0
  8. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/base.py +19 -3
  9. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/cli.py +2 -3
  10. ktoolbox-0.3.0/ktoolbox/configuration.py +157 -0
  11. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/downloader.py +26 -5
  12. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/utils.py +45 -11
  13. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/pyproject.toml +19 -9
  14. ktoolbox-0.2.2/PKG-INFO +0 -272
  15. ktoolbox-0.2.2/README.md +0 -239
  16. ktoolbox-0.2.2/ktoolbox/__main__.py +0 -13
  17. ktoolbox-0.2.2/ktoolbox/configuration.py +0 -134
  18. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/LICENSE +0 -0
  19. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/base.py +0 -0
  20. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/fetch.py +0 -0
  21. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/action/search.py +0 -0
  22. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/__init__.py +0 -0
  23. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/misc/__init__.py +0 -0
  24. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/misc/get_app_version.py +0 -0
  25. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/__init__.py +0 -0
  26. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/announcement.py +0 -0
  27. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/creator.py +0 -0
  28. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/model/post.py +0 -0
  29. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/__init__.py +0 -0
  30. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_announcement.py +0 -0
  31. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_creator_post.py +0 -0
  32. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_creators.py +0 -0
  33. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/posts/get_post.py +0 -0
  34. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/api/utils.py +0 -0
  35. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/__init__.py +0 -0
  36. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/downloader/base.py +0 -0
  37. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/enum.py +0 -0
  38. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/__init__.py +0 -0
  39. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/model.py +0 -0
  40. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/job/runner.py +0 -0
  41. {ktoolbox-0.2.2 → ktoolbox-0.3.0}/ktoolbox/model.py +0 -0
@@ -0,0 +1,191 @@
1
+ Metadata-Version: 2.1
2
+ Name: ktoolbox
3
+ Version: 0.3.0
4
+ Summary: A useful CLI tool for downloading posts in Kemono.party / .su
5
+ Home-page: https://ktoolbox.readthedocs.io/
6
+ Keywords: kemono,kemono.party,cli-app,downloader,os-independent
7
+ Author: Ljzd-PRO
8
+ Author-email: ljzd@office.ljzd-pro.asia
9
+ Requires-Python: >=3.8,<3.12
10
+ Classifier: Development Status :: 5 - Production/Stable
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.8
15
+ Classifier: Programming Language :: Python :: 3.9
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Topic :: Terminals
19
+ Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
20
+ Requires-Dist: fire (>=0.5.0,<0.6.0)
21
+ Requires-Dist: httpx[socks] (>=0.25.2,<0.26.0)
22
+ Requires-Dist: loguru (>=0.7.2,<0.8.0)
23
+ Requires-Dist: pathvalidate (>=3.2.0,<4.0.0)
24
+ Requires-Dist: pydantic (>=2.4.2,<3.0.0)
25
+ Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
26
+ Requires-Dist: tenacity (>=8.2.3,<9.0.0)
27
+ Requires-Dist: tqdm (>=4.66.1,<5.0.0)
28
+ Project-URL: Bug Tracker, https://github.com/Ljzd-PRO/KToolBox/issues
29
+ Project-URL: Documentation, https://ktoolbox.readthedocs.io/
30
+ Project-URL: Repository, https://github.com/Ljzd-PRO/KToolBox
31
+ Description-Content-Type: text/markdown
32
+
33
+ <p align="center" style="text-decoration:none">
34
+ <img align="center" src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/repository-open-graph-2.svg" alt="logo">
35
+ </p>
36
+
37
+ <h1 align="center">
38
+ KToolBox
39
+ </h1>
40
+
41
+ <p align="center">
42
+ KToolBox is a useful CLI tool for downloading posts content in
43
+ <a href="https://kemono.su/">Kemono.party / Kemono.su</a>
44
+ </p>
45
+
46
+ <p align="center">
47
+ <a href="https://pypi.org/project/ktoolbox" target="_blank">
48
+ <img src="https://img.shields.io/github/v/release/Ljzd-PRO/KToolBox?logo=python" alt="Version">
49
+ </a>
50
+
51
+ <a href="./LICENSE">
52
+ <img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
53
+ </a>
54
+
55
+ <a href="https://github.com/Ljzd-PRO/KToolBox/activity">
56
+ <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
57
+ </a>
58
+
59
+ <a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
60
+ <img src="https://codecov.io/gh/Ljzd-PRO/KToolBox/branch/master/graph/badge.svg?token=5XK9CYQHQN" alt="codecov"/>
61
+ </a>
62
+
63
+ <a href='https://ktoolbox.readthedocs.io/'>
64
+ <img src='https://readthedocs.org/projects/ktoolbox/badge/?version=latest' alt='Documentation Status' />
65
+ </a>
66
+
67
+ <a style="text-decoration:none">
68
+ <img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
69
+ </a>
70
+ </p>
71
+
72
+ <p align="center">
73
+ <a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
74
+ </p>
75
+
76
+ <img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
77
+
78
+ ## Features
79
+
80
+ - Support for **multi-thread** downloads (technically, coroutine)
81
+ - **Retry** after download failed
82
+ - Ability to download individual post as well as **all post** by a specified creator/artist
83
+ - **Update downloaded** creator/artist directories to the latest status
84
+ - Customize the **structure** of downloaded post/creator **directories**
85
+ - Search for creators/artists and posts, and **export the results**
86
+ - **Cross-platform** support & **iOS shortcuts** available
87
+
88
+ ## Dev Plan
89
+
90
+ - [ ] Add a Fluent Design style GUI
91
+ - [x] Add uvloop support for Unix platform
92
+
93
+ ## Tutorial
94
+
95
+ See [documentation](https://ktoolbox.readthedocs.io/) for more details.
96
+
97
+ ### Installation
98
+
99
+ - Normal
100
+ ```bash
101
+ pip3 install ktoolbox
102
+ ```
103
+
104
+ - For [a-Shell](https://github.com/holzschu/a-shell)
105
+ ```bash
106
+ pip3 install ktoolbox-pure-py
107
+ ```
108
+
109
+ ### Command
110
+
111
+ For more information, use the help command or goto [Command](https://ktoolbox.readthedocs.io/latest/commands/) page.
112
+
113
+ #### ❓ Get general help
114
+ ```bash
115
+ ktoolbox -h
116
+ ```
117
+
118
+ #### ❓ Get help of a command
119
+ ```bash
120
+ ktoolbox download-post -h
121
+ ```
122
+
123
+ #### ⬇️🖼️ Download a specific post
124
+ ```bash
125
+ ktoolbox download-post https://kemono.su/fanbox/user/49494721/post/6608808
126
+ ```
127
+
128
+ If some files failed to download, you can try to execute the command line again,
129
+ the downloaded files will be **skipped**.
130
+
131
+ #### ⬇️🖌️ Download all posts from a creator
132
+ ```bash
133
+ ktoolbox sync-creator https://kemono.su/fanbox/user/9016
134
+ ```
135
+
136
+ > By default, you will get a `creator-indices.ktoolbox` file in the creator directory,
137
+ > you can use it to update the directory anytime.
138
+
139
+
140
+ #### 🔄️ Update a downloaded creator directory
141
+ ```bash
142
+ ktoolbox sync-creator https://kemono.su/fanbox/user/641955 --update-with=./xxx/creator-indices.ktoolbox
143
+ ```
144
+
145
+ The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.
146
+
147
+ ### iOS Shortcuts
148
+
149
+ Goto [Shortcuts for iOS](https://ktoolbox.readthedocs.io/latest/shortcut/) page for more details.
150
+
151
+ ### Configuration
152
+
153
+ Goto [Configuration-Guide](https://ktoolbox.readthedocs.io/latest/configuration/guide/) page for more details.
154
+
155
+ ## Other Branches
156
+
157
+ - Pure Python branch: [🔗pure-py](https://github.com/Ljzd-PRO/KToolBox/tree/pure-py)
158
+ - Use pydantic v1 so that cargo is not needed for installation
159
+ - For example, you can use it on iOS terminal App [a-Shell](https://github.com/holzschu/a-shell)
160
+ - 🔗[PyPI](https://pypi.org/project/ktoolbox-pure-py/)
161
+ - Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
162
+
163
+ ## About Kemono
164
+
165
+ Description from https://kemono.su :
166
+
167
+ > Kemono is a public archiver for:
168
+ >
169
+ > - Patreon
170
+ > - Pixiv Fanbox
171
+ > - Discord
172
+ > - Fantia
173
+ > - Afdian
174
+ > - Boosty
175
+ > - DLsite
176
+ > - Gumroad
177
+ > - SubscribeStar
178
+ >
179
+ > Contributors here upload content and share it here for easy searching and organization. \
180
+ > To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
181
+
182
+ ## Code Coverage
183
+
184
+ ![codecov.io](https://codecov.io/gh/Ljzd-PRO/KToolBox/graphs/sunburst.svg?token=5XK9CYQHQN)
185
+
186
+ ## License
187
+
188
+ KToolBox is licensed under MIT.
189
+
190
+ Copyright © 2023 by Ljzd-PRO.
191
+
@@ -0,0 +1,158 @@
1
+ <p align="center" style="text-decoration:none">
2
+ <img align="center" src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/repository-open-graph-2.svg" alt="logo">
3
+ </p>
4
+
5
+ <h1 align="center">
6
+ KToolBox
7
+ </h1>
8
+
9
+ <p align="center">
10
+ KToolBox is a useful CLI tool for downloading posts content in
11
+ <a href="https://kemono.su/">Kemono.party / Kemono.su</a>
12
+ </p>
13
+
14
+ <p align="center">
15
+ <a href="https://pypi.org/project/ktoolbox" target="_blank">
16
+ <img src="https://img.shields.io/github/v/release/Ljzd-PRO/KToolBox?logo=python" alt="Version">
17
+ </a>
18
+
19
+ <a href="./LICENSE">
20
+ <img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
21
+ </a>
22
+
23
+ <a href="https://github.com/Ljzd-PRO/KToolBox/activity">
24
+ <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
25
+ </a>
26
+
27
+ <a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
28
+ <img src="https://codecov.io/gh/Ljzd-PRO/KToolBox/branch/master/graph/badge.svg?token=5XK9CYQHQN" alt="codecov"/>
29
+ </a>
30
+
31
+ <a href='https://ktoolbox.readthedocs.io/'>
32
+ <img src='https://readthedocs.org/projects/ktoolbox/badge/?version=latest' alt='Documentation Status' />
33
+ </a>
34
+
35
+ <a style="text-decoration:none">
36
+ <img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
37
+ </a>
38
+ </p>
39
+
40
+ <p align="center">
41
+ <a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
42
+ </p>
43
+
44
+ <img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
45
+
46
+ ## Features
47
+
48
+ - Support for **multi-thread** downloads (technically, coroutine)
49
+ - **Retry** after download failed
50
+ - Ability to download individual post as well as **all post** by a specified creator/artist
51
+ - **Update downloaded** creator/artist directories to the latest status
52
+ - Customize the **structure** of downloaded post/creator **directories**
53
+ - Search for creators/artists and posts, and **export the results**
54
+ - **Cross-platform** support & **iOS shortcuts** available
55
+
56
+ ## Dev Plan
57
+
58
+ - [ ] Add a Fluent Design style GUI
59
+ - [x] Add uvloop support for Unix platform
60
+
61
+ ## Tutorial
62
+
63
+ See [documentation](https://ktoolbox.readthedocs.io/) for more details.
64
+
65
+ ### Installation
66
+
67
+ - Normal
68
+ ```bash
69
+ pip3 install ktoolbox
70
+ ```
71
+
72
+ - For [a-Shell](https://github.com/holzschu/a-shell)
73
+ ```bash
74
+ pip3 install ktoolbox-pure-py
75
+ ```
76
+
77
+ ### Command
78
+
79
+ For more information, use the help command or goto [Command](https://ktoolbox.readthedocs.io/latest/commands/) page.
80
+
81
+ #### ❓ Get general help
82
+ ```bash
83
+ ktoolbox -h
84
+ ```
85
+
86
+ #### ❓ Get help of a command
87
+ ```bash
88
+ ktoolbox download-post -h
89
+ ```
90
+
91
+ #### ⬇️🖼️ Download a specific post
92
+ ```bash
93
+ ktoolbox download-post https://kemono.su/fanbox/user/49494721/post/6608808
94
+ ```
95
+
96
+ If some files failed to download, you can try to execute the command line again,
97
+ the downloaded files will be **skipped**.
98
+
99
+ #### ⬇️🖌️ Download all posts from a creator
100
+ ```bash
101
+ ktoolbox sync-creator https://kemono.su/fanbox/user/9016
102
+ ```
103
+
104
+ > By default, you will get a `creator-indices.ktoolbox` file in the creator directory,
105
+ > you can use it to update the directory anytime.
106
+
107
+
108
+ #### 🔄️ Update a downloaded creator directory
109
+ ```bash
110
+ ktoolbox sync-creator https://kemono.su/fanbox/user/641955 --update-with=./xxx/creator-indices.ktoolbox
111
+ ```
112
+
113
+ The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.
114
+
115
+ ### iOS Shortcuts
116
+
117
+ Goto [Shortcuts for iOS](https://ktoolbox.readthedocs.io/latest/shortcut/) page for more details.
118
+
119
+ ### Configuration
120
+
121
+ Goto [Configuration-Guide](https://ktoolbox.readthedocs.io/latest/configuration/guide/) page for more details.
122
+
123
+ ## Other Branches
124
+
125
+ - Pure Python branch: [🔗pure-py](https://github.com/Ljzd-PRO/KToolBox/tree/pure-py)
126
+ - Use pydantic v1 so that cargo is not needed for installation
127
+ - For example, you can use it on iOS terminal App [a-Shell](https://github.com/holzschu/a-shell)
128
+ - 🔗[PyPI](https://pypi.org/project/ktoolbox-pure-py/)
129
+ - Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
130
+
131
+ ## About Kemono
132
+
133
+ Description from https://kemono.su :
134
+
135
+ > Kemono is a public archiver for:
136
+ >
137
+ > - Patreon
138
+ > - Pixiv Fanbox
139
+ > - Discord
140
+ > - Fantia
141
+ > - Afdian
142
+ > - Boosty
143
+ > - DLsite
144
+ > - Gumroad
145
+ > - SubscribeStar
146
+ >
147
+ > Contributors here upload content and share it here for easy searching and organization. \
148
+ > To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
149
+
150
+ ## Code Coverage
151
+
152
+ ![codecov.io](https://codecov.io/gh/Ljzd-PRO/KToolBox/graphs/sunburst.svg?token=5XK9CYQHQN)
153
+
154
+ ## License
155
+
156
+ KToolBox is licensed under MIT.
157
+
158
+ Copyright © 2023 by Ljzd-PRO.
@@ -1,4 +1,4 @@
1
1
  __title__ = "KToolBox"
2
2
  # noinspection SpellCheckingInspection
3
3
  __description__ = "A useful CLI tool for downloading posts in Kemono.party / .su"
4
- __version__ = "0.2.2"
4
+ __version__ = "0.3.0"
@@ -0,0 +1,20 @@
1
+ import fire
2
+ from loguru import logger
3
+
4
+ from ktoolbox.cli import KToolBoxCli
5
+ from ktoolbox.configuration import config
6
+ from ktoolbox.utils import logger_init, uvloop_init, generate_msg
7
+
8
+
9
+ def main():
10
+ try:
11
+ logger_init(cli_use=True)
12
+ logger.debug(generate_msg(config=config))
13
+ uvloop_init()
14
+ fire.Fire(KToolBoxCli)
15
+ except KeyboardInterrupt:
16
+ logger.error("KToolBox was interrupted by the user")
17
+
18
+
19
+ if __name__ == "__main__":
20
+ main()
@@ -2,3 +2,4 @@ from .base import *
2
2
  from .fetch import *
3
3
  from .job import *
4
4
  from .search import *
5
+ from .utils import *
@@ -6,6 +6,7 @@ from loguru import logger
6
6
  from pathvalidate import sanitize_filename
7
7
 
8
8
  from ktoolbox.action import ActionRet, fetch_all_creator_posts, FetchInterruptError
9
+ from ktoolbox.action.utils import generate_post_path_name
9
10
  from ktoolbox.api.model import Post
10
11
  from ktoolbox.api.posts import get_creator_post
11
12
  from ktoolbox.configuration import config, PostStructureConfiguration
@@ -47,7 +48,7 @@ async def create_job_from_post(
47
48
 
48
49
  # Create jobs
49
50
  jobs: List[Job] = []
50
- for attachment in post.attachments: # attachments
51
+ for attachment in post.attachments or []: # attachments
51
52
  if not attachment.path:
52
53
  continue
53
54
  jobs.append(
@@ -58,7 +59,7 @@ async def create_job_from_post(
58
59
  type=PostFileTypeEnum.Attachment
59
60
  )
60
61
  )
61
- if post.file.path: # file
62
+ if post.file and post.file.path: # file
62
63
  jobs.append(
63
64
  Job(
64
65
  path=post_path,
@@ -170,12 +171,7 @@ async def create_job_from_creator(
170
171
  job_list: List[Job] = []
171
172
  for post in post_list:
172
173
  # Get post path
173
- if mix_posts:
174
- default_post_path = path
175
- elif config.job.post_id_as_path:
176
- default_post_path = path / post.id
177
- else:
178
- default_post_path = path / sanitize_filename(post.title)
174
+ default_post_path = path if mix_posts else path / generate_post_path_name(post)
179
175
  if update_from:
180
176
  if not (post_path := update_from.posts_path.get(post.id)):
181
177
  post_path = default_post_path
@@ -0,0 +1,14 @@
1
+ from pathvalidate import sanitize_filename
2
+
3
+ from ktoolbox.api.model import Post
4
+ from ktoolbox.configuration import config
5
+
6
+ __all__ = ["generate_post_path_name"]
7
+
8
+
9
+ def generate_post_path_name(post: Post) -> str:
10
+ """Generate directory name for post to save."""
11
+ if config.job.post_id_as_path or not post.title:
12
+ return post.id
13
+ else:
14
+ return sanitize_filename(post.title)
@@ -1,16 +1,17 @@
1
1
  from abc import ABC, abstractmethod
2
- from typing import Literal, Generic, TypeVar, Optional, Callable, Any
2
+ from typing import Literal, Generic, TypeVar, Optional, Callable
3
3
  from urllib.parse import urlunparse
4
4
 
5
5
  import httpx
6
6
  import tenacity
7
+ from loguru import logger
7
8
  from pydantic import BaseModel, ValidationError, RootModel
8
9
  from tenacity import RetryCallState, wait_fixed, retry_if_result
9
10
  from tenacity.stop import stop_base, stop_never, stop_after_attempt
10
11
 
11
12
  from ktoolbox.configuration import config
12
13
  from ktoolbox.enum import RetCodeEnum
13
- from ktoolbox.utils import BaseRet
14
+ from ktoolbox.utils import BaseRet, generate_msg
14
15
 
15
16
  __all__ = ["APITenacityStop", "APIRet", "BaseAPI"]
16
17
 
@@ -27,6 +28,21 @@ class APITenacityStop(stop_base):
27
28
  return stop_after_attempt(config.api.retry_times)(retry_state)
28
29
 
29
30
 
31
+ def _retry_error_callback(state: RetryCallState) -> "APIRet":
32
+ """
33
+ Call after all reties failed
34
+ :return Keep the origin return value
35
+ """
36
+ # noinspection SpellCheckingInspection
37
+ logger.error(
38
+ generate_msg(
39
+ f"Kemono API call failed",
40
+ ret=state.outcome.result(),
41
+ )
42
+ )
43
+ return state.outcome.result()
44
+
45
+
30
46
  def _retry(*args, **kwargs):
31
47
  """Wrap an API method with a new ``Retrying`` object"""
32
48
  wrapper = tenacity.retry(
@@ -34,7 +50,7 @@ def _retry(*args, **kwargs):
34
50
  wait=wait_fixed(config.api.retry_interval),
35
51
  retry=retry_if_result(lambda x: not bool(x)),
36
52
  reraise=True,
37
- retry_error_callback=lambda x: x.outcome.result(), # type: Callable[[RetryCallState], Any]
53
+ retry_error_callback=_retry_error_callback,
38
54
  **kwargs
39
55
  )
40
56
  if len(args) == 1 and callable(args[0]):
@@ -6,7 +6,7 @@ from loguru import logger
6
6
  from pathvalidate import sanitize_filename
7
7
 
8
8
  from ktoolbox import __version__
9
- from ktoolbox.action import create_job_from_post, create_job_from_creator
9
+ from ktoolbox.action import create_job_from_post, create_job_from_creator, generate_post_path_name
10
10
  from ktoolbox.action import search_creator as search_creator_action, search_creator_post as search_creator_post_action
11
11
  from ktoolbox.api.misc import get_app_version
12
12
  from ktoolbox.api.posts import get_post as get_post_api
@@ -171,7 +171,7 @@ class KToolBoxCli:
171
171
  post_id=post_id
172
172
  )
173
173
  if ret:
174
- post_path = path / ret.data.id if config.job.post_id_as_path else path / sanitize_filename(ret.data.title)
174
+ post_path = path / generate_post_path_name(ret.data)
175
175
  job_list = await create_job_from_post(
176
176
  post=ret.data,
177
177
  post_path=post_path,
@@ -224,7 +224,6 @@ class KToolBoxCli:
224
224
  You can update the directory anytime after download finished, \
225
225
  such as to update after creator published new posts.
226
226
 
227
- .. rubric:: NOTICE:
228
227
  * If ``update_from`` was provided, it should be located **inside the creator directory**.
229
228
 
230
229
  :param url: The post URL
@@ -0,0 +1,157 @@
1
+ import datetime
2
+ import logging
3
+ from pathlib import Path
4
+ from typing import Literal, Union, Optional
5
+
6
+ from pydantic import BaseModel
7
+ from pydantic_settings import BaseSettings, SettingsConfigDict
8
+
9
+ __all__ = [
10
+ "config",
11
+ "APIConfiguration",
12
+ "DownloaderConfiguration",
13
+ "PostStructureConfiguration",
14
+ "JobConfiguration",
15
+ "Configuration"
16
+ ]
17
+
18
+
19
+ # noinspection SpellCheckingInspection,GrazieInspection
20
+ class APIConfiguration(BaseModel):
21
+ """
22
+ Kemono API Configuration
23
+
24
+ :ivar scheme: Kemono API URL scheme
25
+ :ivar netloc: Kemono API URL netloc
26
+ :ivar statics_netloc: URL netloc of Kemono server for static files (e.g. images)
27
+ :ivar files_netloc: URL netloc of Kemono server for post files
28
+ :ivar path: Kemono API URL root path
29
+ :ivar timeout: API request timeout
30
+ :ivar retry_times: API request retry times (when request failed)
31
+ :ivar retry_interval: Seconds of API request retry interval
32
+ """
33
+ scheme: Literal["http", "https"] = "https"
34
+ netloc: str = "kemono.su"
35
+ statics_netloc: str = "img.kemono.su"
36
+ files_netloc: str = "kemono.su"
37
+ path: str = "/api/v1"
38
+ timeout: float = 5.0
39
+ retry_times: int = 3
40
+ retry_interval: float = 2.0
41
+
42
+
43
+ class DownloaderConfiguration(BaseModel):
44
+ """
45
+ File Downloader Configuration
46
+
47
+ :ivar scheme: Downloader URL scheme
48
+ :ivar timeout: Downloader request timeout
49
+ :ivar encoding: Charset for filename parsing and post content text saving
50
+ :ivar buffer_size: Number of bytes of file I/O buffer for each downloading file
51
+ :ivar chunk_size: Number of bytes of chunk of downloader stream
52
+ :ivar temp_suffix: Temp filename suffix of downloading files
53
+ :ivar retry_times: Downloader retry times (when download failed)
54
+ :ivar retry_stop_never: Never stop downloader from retrying (when download failed) \
55
+ (``retry_times`` will be ignored when enabled)
56
+ :ivar retry_interval: Seconds of downloader retry interval
57
+ """
58
+ scheme: Literal["http", "https"] = "https"
59
+ timeout: float = 30.0
60
+ encoding: str = "utf-8"
61
+ buffer_size: int = 20480
62
+ chunk_size: int = 1024
63
+ temp_suffix: str = "tmp"
64
+ retry_times: int = 10
65
+ retry_stop_never: bool = False
66
+ retry_interval: float = 3.0
67
+
68
+
69
+ class PostStructureConfiguration(BaseModel):
70
+ # noinspection SpellCheckingInspection
71
+ """
72
+ Post path structure model
73
+
74
+ - Default:
75
+ ```
76
+ |-- ..
77
+ |-- attachments
78
+ | |-- 1.png
79
+ | |-- 2.png
80
+ |-- content.txt
81
+ |-- <Post file>
82
+ |-- <Post data (post.ktoolbox.json)>
83
+ ```
84
+
85
+ :ivar attachments: Sub path of attachment directory
86
+ :ivar content_filepath: Sub path of post content HTML file
87
+ """
88
+ attachments: Path = Path("attachments")
89
+ content_filepath: Path = Path("index.html")
90
+
91
+
92
+ class JobConfiguration(BaseModel):
93
+ """
94
+ Download jobs Configuration
95
+
96
+ :ivar count: Number of coroutines for concurrent download
97
+ :ivar post_id_as_path: Use post ID as post directory name
98
+ :ivar post_structure: Post path structure
99
+ :ivar mix_posts: Save all files from different posts at same path in creator directory. \
100
+ It would not create any post directory, and ``CreatorIndices`` would not been recorded, \
101
+ without ``CreatorIndices`` you **cannot update** the creator directory.
102
+ :ivar job_list_filepath: Filepath for job list data saving, ``None`` for disable job list saving
103
+ """
104
+ count: int = 4
105
+ post_id_as_path: bool = False
106
+ post_structure: PostStructureConfiguration = PostStructureConfiguration()
107
+ mix_posts: bool = False
108
+ job_list_filepath: Optional[Path] = None
109
+
110
+
111
+ class LoggerConfiguration(BaseModel):
112
+ """
113
+ Logger configuration
114
+
115
+ :ivar path: Path to save logs, ``None`` for disable log file output
116
+ :ivar level: Log filter level
117
+ :ivar rotation: Log rotation
118
+ """
119
+ path: Optional[Path] = Path("logs")
120
+ level: Union[str, int] = logging.DEBUG
121
+ rotation: Union[str, int, datetime.time, datetime.timedelta] = "1 week"
122
+
123
+
124
+ class Configuration(BaseSettings):
125
+ # noinspection SpellCheckingInspection,GrazieInspection
126
+ """
127
+ KToolBox Configuration
128
+
129
+ :ivar api:
130
+ :ivar downloader:
131
+ :ivar job:
132
+ :ivar logger:
133
+ :ivar ssl_verify: Enable SSL certificate verification for Kemono API server and download server
134
+ :ivar json_dump_indent: Indent of JSON file dump
135
+ :ivar use_uvloop: Use uvloop for asyncio (Disabled on Windows by default) \
136
+ uvloop will improve concurrent performance, but it is not compatible with Windows. \
137
+ Install uvloop by `pip install ktoolbox[uvloop]` or it will not work.
138
+ """
139
+ api: APIConfiguration = APIConfiguration()
140
+ downloader: DownloaderConfiguration = DownloaderConfiguration()
141
+ job: JobConfiguration = JobConfiguration()
142
+ logger: LoggerConfiguration = LoggerConfiguration()
143
+
144
+ ssl_verify: bool = True
145
+ json_dump_indent: int = 4
146
+ use_uvloop: bool = True
147
+
148
+ # noinspection SpellCheckingInspection
149
+ model_config = SettingsConfigDict(
150
+ env_prefix='ktoolbox_',
151
+ env_nested_delimiter='__',
152
+ env_file='.env',
153
+ env_file_encoding='utf-8'
154
+ )
155
+
156
+
157
+ config = Configuration(_env_file='prod.env')