ktoolbox 0.1.0.dev0__tar.gz → 0.2.1__tar.gz

ktoolbox-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Ljzd PRO
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: ktoolbox
-Version: 0.1.0.dev0
-Summary: A useful tool for Kemono.party
+Version: 0.2.1
+Summary: A useful CLI tool for downloading posts in Kemono.party / .su
 Home-page: https://github.com/Ljzd-PRO/KToolBox
 Keywords: kemono,kemono.party,cli-app,downloader,os-independent
 Author: Ljzd-PRO
@@ -20,6 +20,7 @@ Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
 Requires-Dist: fire (>=0.5.0,<0.6.0)
 Requires-Dist: httpx (<=0.25.0)
 Requires-Dist: loguru (>=0.7.2,<0.8.0)
+Requires-Dist: pathvalidate (>=3.2.0,<4.0.0)
 Requires-Dist: pydantic (>=2.4.2,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)
@@ -29,6 +30,10 @@ Project-URL: Documentation, https://github.com/Ljzd-PRO/KToolBox/wiki
 Project-URL: Repository, https://github.com/Ljzd-PRO/KToolBox
 Description-Content-Type: text/markdown
 
+<p align="center" style="text-decoration:none">
+  <img align="center" src="https://raw.githubusercontent.com/Ljzd-PRO/KToolBox/master/static/repository-open-graph-2.svg" alt="logo">
+</p>
+
 <h1 align="center">
   KToolBox
 </h1>
@@ -44,11 +49,11 @@ Description-Content-Type: text/markdown
   </a>
 
   <a href="./LICENSE">
-    <img src="https://img.shields.io/badge/License-GPLv3-blue" alt="GPLv3"/>
+    <img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
   </a>
 
-  <a href="https://github.com/Ljzd-PRO/KToolBox/commits/master">
-    <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/master" alt="Last Commit"/>
+  <a href="https://github.com/Ljzd-PRO/KToolBox/activity">
+    <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
   </a>
 
   <a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
@@ -56,7 +61,7 @@ Description-Content-Type: text/markdown
   </a>
 
   <a style="text-decoration:none">
-    <img src="https://img.shields.io/badge/Platform-Win%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
+    <img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
   </a>
 </p>
 
@@ -64,7 +69,7 @@ Description-Content-Type: text/markdown
     <a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
 </p>
 
-![Preview](./static/preview-1.png)
+<img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
 
 ## Features
 
@@ -83,9 +88,11 @@ Description-Content-Type: text/markdown
 
 ## Tutorial
 
+### Command
+
 For more information, use the help command.
 
-> **Warning**
+> [!Warning]
 > The return text here are only **demo**, some of them may be **outdated**.
 
 - Install KToolBox:
@@ -181,6 +188,35 @@ For more information, use the help command.
   
   The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.  
 
+### Configuration
+
+- KToolBox read **`prod.env` file** in the work folder or **environment variables** to store configuration
+- **All configuration options** defined in [`ktoolbox/config.py`](ktoolbox/configuration.py)
+- Use `__` to specify the sub option, like `KTOOLBOX_API__SCHEME` means `api.scheme`
+- All configuration options are optional
+
+#### `prod.env` file example
+
+```dotenv
+# Download 10 files at the same time.
+KTOOLBOX_JOB__COUNT=10
+
+# Allocate 102400 Bytes as buffer for each download job
+KTOOLBOX_DOWNLOADER__BUFFER_SIZE=102400
+
+# Set post attachments directory path as `./`, it means to save all attachments files in post directory
+# without making a new sub directory to storage them
+KTOOLBOX_JOB__POST_STRUCTURE__ATTACHMENTS=./
+
+# Disable SSL certificate verification for Kemono API server and download server
+# It's useful when certificate on Kemono server expired. (SSL: CERTIFICATE_VERIFY_FAILED)
+KTOOLBOX_SSL_VERIFY=False
+```
+
+## About Development
+
+- Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
+
 ## About Kemono
 
 Description from https://kemono.su :
@@ -200,9 +236,13 @@ Description from https://kemono.su :
 > Contributors here upload content and share it here for easy searching and organization. \
 > To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
 
+## Code Coverage
+
+![codecov.io](https://codecov.io/gh/Ljzd-PRO/KToolBox/graphs/sunburst.svg?token=5XK9CYQHQN)
+
 ## License
 
-KToolBox is licensed under GPLv3.
+KToolBox is licensed under MIT.
 
 Copyright © 2023 by Ljzd-PRO.
 

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/README.md

@@ -1,3 +1,7 @@
+<p align="center" style="text-decoration:none">
+  <img align="center" src="https://raw.githubusercontent.com/Ljzd-PRO/KToolBox/master/static/repository-open-graph-2.svg" alt="logo">
+</p>
+
 <h1 align="center">
   KToolBox
 </h1>
@@ -13,11 +17,11 @@
   </a>
 
   <a href="./LICENSE">
-    <img src="https://img.shields.io/badge/License-GPLv3-blue" alt="GPLv3"/>
+    <img src="https://img.shields.io/badge/License-MIT-blue" alt="MIT"/>
   </a>
 
-  <a href="https://github.com/Ljzd-PRO/KToolBox/commits/master">
-    <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/master" alt="Last Commit"/>
+  <a href="https://github.com/Ljzd-PRO/KToolBox/activity">
+    <img src="https://img.shields.io/github/last-commit/Ljzd-PRO/KToolBox/devel" alt="Last Commit"/>
   </a>
 
   <a href="https://codecov.io/gh/Ljzd-PRO/KToolBox" target="_blank">
@@ -25,7 +29,7 @@
   </a>
 
   <a style="text-decoration:none">
-    <img src="https://img.shields.io/badge/Platform-Win%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
+    <img src="https://img.shields.io/badge/Platform-Windows%20|%20Linux%20|%20macOS-blue" alt="Platform Win | Linux | macOS"/>
   </a>
 </p>
 
@@ -33,7 +37,7 @@
     <a href="./README.md">English</a> | <a href="./README_zh-CN.md">中文</a>
 </p>
 
-![Preview](./static/preview-1.png)
+<img src="https://cdn.jsdelivr.net/gh/Ljzd-PRO/KToolBox@latest/static/preview-1.png" alt="Preview">
 
 ## Features
 
@@ -52,9 +56,11 @@
 
 ## Tutorial
 
+### Command
+
 For more information, use the help command.
 
-> **Warning**
+> [!Warning]
 > The return text here are only **demo**, some of them may be **outdated**.
 
 - Install KToolBox:
@@ -150,6 +156,35 @@ For more information, use the help command.
   
   The `creator-indices.ktoolbox` file contains the information and filepath of posts inside the directory.  
 
+### Configuration
+
+- KToolBox read **`prod.env` file** in the work folder or **environment variables** to store configuration
+- **All configuration options** defined in [`ktoolbox/config.py`](ktoolbox/configuration.py)
+- Use `__` to specify the sub option, like `KTOOLBOX_API__SCHEME` means `api.scheme`
+- All configuration options are optional
+
+#### `prod.env` file example
+
+```dotenv
+# Download 10 files at the same time.
+KTOOLBOX_JOB__COUNT=10
+
+# Allocate 102400 Bytes as buffer for each download job
+KTOOLBOX_DOWNLOADER__BUFFER_SIZE=102400
+
+# Set post attachments directory path as `./`, it means to save all attachments files in post directory
+# without making a new sub directory to storage them
+KTOOLBOX_JOB__POST_STRUCTURE__ATTACHMENTS=./
+
+# Disable SSL certificate verification for Kemono API server and download server
+# It's useful when certificate on Kemono server expired. (SSL: CERTIFICATE_VERIFY_FAILED)
+KTOOLBOX_SSL_VERIFY=False
+```
+
+## About Development
+
+- Development branch: [🔗devel](https://github.com/Ljzd-PRO/KToolBox/tree/devel)
+
 ## About Kemono
 
 Description from https://kemono.su :
@@ -169,8 +204,12 @@ Description from https://kemono.su :
 > Contributors here upload content and share it here for easy searching and organization. \
 > To get started viewing content, either search for creators on the artists page, or search for content on the posts page.
 
+## Code Coverage
+
+![codecov.io](https://codecov.io/gh/Ljzd-PRO/KToolBox/graphs/sunburst.svg?token=5XK9CYQHQN)
+
 ## License
 
-KToolBox is licensed under GPLv3.
+KToolBox is licensed under MIT.
 
 Copyright © 2023 by Ljzd-PRO.

ktoolbox-0.2.1/ktoolbox/__init__.py

@@ -0,0 +1,4 @@
+__title__ = "KToolBox"
+# noinspection SpellCheckingInspection
+__description__ = "A useful CLI tool for downloading posts in Kemono.party / .su"
+__version__ = "0.2.1"

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/__main__.py

@@ -1,10 +1,13 @@
-import logging
-
 import fire
 
 from ktoolbox.cli import KToolBoxCli
 from ktoolbox.utils import logger_init
 
-if __name__ == "__main__":
-    logger_init(std_level=logging.WARNING)
+
+def main():
+    logger_init(cli_use=True)
     fire.Fire(KToolBoxCli)
+
+
+if __name__ == "__main__":
+    main()

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/action/job.py

@@ -2,6 +2,8 @@ from pathlib import Path
 from typing import List, Union, Tuple
 
 import aiofiles
+from loguru import logger
+from pathvalidate import sanitize_filename
 
 from ktoolbox.action import ActionRet, fetch_all_creator_posts, FetchInterruptError
 from ktoolbox.api.model import Post
@@ -29,16 +31,25 @@ async def create_job_from_post(
      ``True`` & ``None`` -> ``config.job.post_structure``
     :param dump_post_data: Whether to dump post data (post.json) in post directory
     """
-    jobs: List[Job] = []
+    post_path.mkdir(exist_ok=True)
+
+    # Load ``PostStructureConfiguration``
     if post_structure in [True, None]:
         post_structure = config.job.post_structure
     if post_structure:
-        attachments_path = post_path / post_structure.attachments
-        content_path = post_path / post_structure.content_filepath
+        attachments_path = post_path / post_structure.attachments  # attachments
+        attachments_path.mkdir(exist_ok=True)
+        content_path = post_path / post_structure.content_filepath  # content
+        content_path.parent.mkdir(exist_ok=True)
     else:
         attachments_path = post_path
         content_path = None
-    for attachment in post.attachments:
+
+    # Create jobs
+    jobs: List[Job] = []
+    for attachment in post.attachments:  # attachments
+        if not attachment.path:
+            continue
         jobs.append(
             Job(
                 path=attachments_path,
@@ -47,17 +58,18 @@ async def create_job_from_post(
                 type=PostFileTypeEnum.Attachment
             )
         )
-    jobs.append(
-        Job(
-            path=post_path,
-            alt_filename=post.file.name,
-            server_path=post.file.path,
-            type=PostFileTypeEnum.File
+    if post.file.path:  # file
+        jobs.append(
+            Job(
+                path=post_path,
+                alt_filename=f"{post.id}_{post.file.name}",
+                server_path=post.file.path,
+                type=PostFileTypeEnum.File
+            )
         )
-    )
-    if content_path:
-        if not content_path.parent.is_dir():
-            content_path.parent.mkdir()
+
+    # Write content file
+    if content_path and post.content:
         async with aiofiles.open(content_path, "w", encoding=config.downloader.encoding) as f:
             await f.write(post.content)
     if dump_post_data:
@@ -65,6 +77,7 @@ async def create_job_from_post(
             await f.write(
                 post.model_dump_json(indent=config.json_dump_indent)
             )
+
     return jobs
 
 
@@ -117,6 +130,7 @@ async def create_job_from_creator(
     mix_posts = config.job.mix_posts if mix_posts is None else mix_posts
 
     # Get posts
+    logger.info(f"Start fetching posts from creator {creator_id}")
     if all_pages:
         post_list: List[Post] = []
         try:
@@ -130,18 +144,20 @@ async def create_job_from_creator(
             post_list = ret.data
         else:
             return ActionRet(**ret.model_dump(mode="python"))
+    logger.info(f"Get {len(post_list)} posts, start creating jobs")
 
     # Filter posts and generate ``CreatorIndices``
     if not mix_posts:
         indices = None
         if update_from:
             post_list, indices = filter_posts_with_indices(post_list, update_from)
+            logger.info(f"{len(post_list)} posts will be downloaded")
         elif save_creator_indices:  # It's unnecessary to create indices again when ``update_from`` was provided
             indices = CreatorIndices(
                 creator_id=creator_id,
                 service=service,
                 posts={post.id: post for post in post_list},
-                posts_path={post.id: path / post.title for post in post_list}
+                posts_path={post.id: path / sanitize_filename(post.title) for post in post_list}
             )
         if indices:
             async with aiofiles.open(
@@ -154,7 +170,7 @@ async def create_job_from_creator(
     job_list: List[Job] = []
     for post in post_list:
         # Get post path
-        default_post_path = path if mix_posts else path / post.title
+        default_post_path = path if mix_posts else path / sanitize_filename(post.title)
         if update_from:
             if not (post_path := update_from.posts_path.get(post.id)):
                 post_path = default_post_path

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/api/base.py

@@ -94,7 +94,7 @@ class BaseAPI(ABC, Generic[_T]):
         url_parts = [config.api.scheme, config.api.netloc, f"{config.api.path}{path}", '', '', '']
         url = urlunparse(url_parts)
         try:
-            async with httpx.AsyncClient() as client:
+            async with httpx.AsyncClient(verify=config.ssl_verify) as client:
                 res = await client.request(
                     method=cls.method,
                     url=url,

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/cli.py

@@ -3,6 +3,7 @@ from typing import Union, overload
 
 import aiofiles
 from loguru import logger
+from pathvalidate import sanitize_filename
 
 from ktoolbox import __version__
 from ktoolbox.action import create_job_from_post, create_job_from_creator
@@ -173,7 +174,7 @@ class KToolBoxCli:
         if ret:
             job_list = await create_job_from_post(
                 post=ret.data,
-                post_path=path / ret.data.title,
+                post_path=path / sanitize_filename(ret.data.title),
                 dump_post_data=dump_post_data
             )
             job_runner = JobRunner(job_list=job_list)
@@ -267,6 +268,13 @@ class KToolBoxCli:
                 creator = next(creator_ret.data, None)
                 if creator:
                     creator_name = creator.name
+                    logger.info(
+                        generate_msg(
+                            "Got creator information",
+                            name=creator.name,
+                            id=creator.id
+                        )
+                    )
             else:
                 logger.warning(
                     generate_msg(
@@ -274,7 +282,7 @@ class KToolBoxCli:
                         detail=creator_ret.message
                     )
                 )
-            creator_path = path / creator_name
+            creator_path = path / sanitize_filename(creator_name)
 
         creator_path.mkdir(exist_ok=True)
         ret = await create_job_from_creator(

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/configuration.py

@@ -116,6 +116,9 @@ class Configuration(BaseSettings):
     job: JobConfiguration = JobConfiguration()
     logger: LoggerConfiguration = LoggerConfiguration()
 
+    ssl_verify: bool = True
+    # noinspection SpellCheckingInspection
+    """Enable SSL certificate verification for Kemono API server and download server"""
     json_dump_indent: int = 4
     """Indent of JSON file dump"""
 

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/downloader/downloader.py

@@ -113,9 +113,20 @@ class Downloader:
         :return: ``DownloaderRet`` which contain the actual output filename
         :raise CancelledError
         """
+        # Use ``self._alt_filename`` instead of filename from server,
+        # to make the process more efficient
+        if (self._path / self._filename).is_file():
+            return DownloaderRet(
+                code=RetCodeEnum.FileExisted,
+                message=generate_msg(
+                    "Download file existed, skipped",
+                    path=self._path / self._filename
+                )
+            )
+
         tqdm_class: Type[std_tqdm] = tqdm_class or tqdm.asyncio.tqdm
         async with self._lock:
-            async with httpx.AsyncClient() as client:
+            async with httpx.AsyncClient(verify=config.ssl_verify) as client:
                 async with client.stream(
                         method="GET",
                         url=self._url,
@@ -138,15 +149,6 @@ class Downloader:
                             filename = urllib.parse.unquote(Path(self._url).name)
                     self._filename = filename
 
-                    if (self._path / filename).is_file():
-                        return DownloaderRet(
-                            code=RetCodeEnum.FileExisted,
-                            message=generate_msg(
-                                "Download file existed, skipped",
-                                path=self._path / filename
-                            )
-                        )
-
                     # Download
                     temp_filepath = (self._path / filename).with_suffix(f".{config.downloader.temp_suffix}")
                     total_size = int(length_str) if (length_str := res.headers.get("Content-Length")) else None

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/job/runner.py

@@ -74,8 +74,6 @@ class JobRunner:
         failed_num = 0
         while not self._job_queue.empty():
             job = await self._job_queue.get()
-            if not job.path.is_dir():
-                job.path.mkdir()
 
             # Create downloader
             url_parts = [config.downloader.scheme, config.api.files_netloc, job.server_path, '', '', '']

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/ktoolbox/utils.py

@@ -1,8 +1,9 @@
 import cgi
+import logging
 import sys
 import urllib.parse
 from pathlib import Path
-from typing import Generic, TypeVar, Optional, Dict, List, Union, Tuple
+from typing import Generic, TypeVar, Optional, Dict, List, Tuple
 
 import aiofiles
 from loguru import logger
@@ -103,31 +104,31 @@ def generate_msg(title: str = None, **kwargs):
     return f"{title} - {kwargs}" if kwargs else title
 
 
-def logger_init(std_level: Union[str, int] = None, disable_stdout: bool = False):
+def logger_init(cli_use: bool = False, disable_stdout: bool = False):
     """
     Initialize ``loguru`` logger
 
-    :param std_level: Standard output log filter level
+    :param cli_use: Set logger level ``INFO`` and filter out ``SUCCESS``
     :param disable_stdout: Disable default output stream
     """
     if disable_stdout:
         logger.remove()
-    elif std_level is not None:
+    elif cli_use:
         logger.remove()
         logger.add(
             sys.stderr,
-            level=std_level
-        )
-    path = config.logger.path
-    if not path.is_dir():
-        path.mkdir()
-    if path is not None:
-        logger.add(
-            path / DataStorageNameEnum.LogData.value,
-            level=config.logger.level,
-            rotation=config.logger.rotation,
-            diagnose=True
+            level=logging.INFO,
+            filter=lambda record: record["level"].name != "SUCCESS"
         )
+    if path := config.logger.path:
+        path.mkdir(exist_ok=True)
+        if path is not None:
+            logger.add(
+                path / DataStorageNameEnum.LogData.value,
+                level=config.logger.level,
+                rotation=config.logger.rotation,
+                diagnose=True
+            )
 
 
 async def dump_search(result: List[BaseModel], path: Path):

{ktoolbox-0.1.0.dev0 → ktoolbox-0.2.1}/pyproject.toml

@@ -1,12 +1,13 @@
 [tool.poetry]
 name = "ktoolbox"
-version = "0.1.0-dev"
-description = "A useful tool for Kemono.party"
+version = "0.2.1"
+description = "A useful CLI tool for downloading posts in Kemono.party / .su"
 authors = ["Ljzd-PRO <ljzd@office.ljzd-pro.asia>"]
 readme = "README.md"
 homepage = "https://github.com/Ljzd-PRO/KToolBox"
 repository = "https://github.com/Ljzd-PRO/KToolBox"
 documentation = "https://github.com/Ljzd-PRO/KToolBox/wiki"
+
 keywords = ["kemono", "kemono.party", "cli-app", "downloader", "os-independent"]
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -21,6 +22,9 @@ packages = [{ include = "ktoolbox" }]
 [tool.poetry.urls]
 "Bug Tracker" = "https://github.com/Ljzd-PRO/KToolBox/issues"
 
+[tool.poetry.scripts]
+ktoolbox = "ktoolbox.__main__:main"
+
 [tool.poetry.dependencies]
 python = ">=3.8,<3.12"
 pydantic = "^2.4.2"
@@ -31,6 +35,7 @@ fire = "^0.5.0"
 tqdm = "^4.66.1"
 loguru = "^0.7.2"
 aiofiles = "^23.2.1"
+pathvalidate = "^3.2.0"
 
 #[tool.poetry.group.uvloop.dependencies]
 #uvloop = "^0.17.0"
@@ -46,7 +51,7 @@ pytest-cov = "==4.1.0"
 allpairspy = "==2.5.1"
 
 [tool.poetry.group.dev.dependencies]
-datamodel-code-generator = "^0.22.1"
+datamodel-code-generator = ">=0.22.1,<0.24.0"
 
 #[tool.poetry.group.uvloop]
 #optional = true