novel-downloader 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

novel_downloader/__init__.py

@@ -7,7 +7,7 @@ novel_downloader
 Core package for the Novel Downloader project.
 """
 
-__version__ = "1.1.1"
+__version__ = "1.2.0"
 
 __author__ = "Saudade Z"
 __email__ = "saudadez217@gmail.com"

novel_downloader/cli/download.py

@@ -14,11 +14,15 @@ import click
 from click import Context
 
 from novel_downloader.config import ConfigAdapter, load_config
-from novel_downloader.core import (
-    get_downloader,
+from novel_downloader.core.factory import (
+    get_async_downloader,
+    get_async_requester,
+    # get_downloader,
     get_parser,
-    get_requester,
+    # get_requester,
     get_saver,
+    get_sync_downloader,
+    get_sync_requester,
 )
 from novel_downloader.utils.i18n import t
 from novel_downloader.utils.logger import setup_logging
@@ -75,24 +79,54 @@ def download_cli(ctx: Context, book_ids: List[str], site: str) -> None:
         return
 
     # Initialize the requester, parser, saver, and downloader components
-    curr_requester = get_requester(site, requester_cfg)
-    curr_parser = get_parser(site, parser_cfg)
-    curr_saver = get_saver(site, saver_cfg)
-    setup_logging()
-    curr_downloader = get_downloader(
-        requester=curr_requester,
-        parser=curr_parser,
-        saver=curr_saver,
-        site=site,
-        config=downloader_cfg,
-    )
-
-    # Perform the download for each valid book ID
-    for book_id in book_ids:
-        click.echo(t("download_downloading", book_id=book_id, site=site))
-        curr_downloader.download_one(book_id)
-
-    # Prompt for parsing and wait for user input before shutting down
-    if requester_cfg.auto_close:
-        input(t("download_prompt_parse"))
-        curr_requester.shutdown()
+    if downloader_cfg.mode == "async":
+        import asyncio
+
+        async_requester = get_async_requester(site, requester_cfg)
+        async_parser = get_parser(site, parser_cfg)
+        async_saver = get_saver(site, saver_cfg)
+        setup_logging()
+        async_downloader = get_async_downloader(
+            requester=async_requester,
+            parser=async_parser,
+            saver=async_saver,
+            site=site,
+            config=downloader_cfg,
+        )
+
+        async def async_download_all() -> None:
+            prepare = getattr(async_downloader, "prepare", None)
+            if prepare and asyncio.iscoroutinefunction(prepare):
+                await prepare()
+
+            for book_id in valid_book_ids:
+                click.echo(t("download_downloading", book_id=book_id, site=site))
+                await async_downloader.download_one(book_id)
+
+            if requester_cfg.auto_close:
+                input(t("download_prompt_parse"))
+                await async_requester.shutdown()
+
+        asyncio.run(async_download_all())
+    else:
+        sync_requester = get_sync_requester(site, requester_cfg)
+        sync_parser = get_parser(site, parser_cfg)
+        sync_saver = get_saver(site, saver_cfg)
+        setup_logging()
+        sync_downloader = get_sync_downloader(
+            requester=sync_requester,
+            parser=sync_parser,
+            saver=sync_saver,
+            site=site,
+            config=downloader_cfg,
+        )
+
+        for book_id in book_ids:
+            click.echo(t("download_downloading", book_id=book_id, site=site))
+            sync_downloader.download_one(book_id)
+
+        if requester_cfg.auto_close:
+            input(t("download_prompt_parse"))
+            sync_requester.shutdown()
+
+    return

novel_downloader/config/adapter.py

@@ -63,6 +63,7 @@ class ConfigAdapter:
             disable_images=req.get("disable_images", True),
             mute_audio=req.get("mute_audio", True),
             mode=site_cfg.get("mode", "session"),
+            max_rps=site_cfg.get("max_rps", None),
         )
 
     def get_downloader_config(self) -> DownloaderConfig:
@@ -77,7 +78,9 @@ class ConfigAdapter:
             request_interval=gen.get("request_interval", 5),
             raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
             cache_dir=gen.get("cache_dir", "./cache"),
-            max_threads=gen.get("max_threads", 4),
+            download_workers=gen.get("download_workers", 4),
+            parser_workers=gen.get("parser_workers", 4),
+            use_process_pool=gen.get("use_process_pool", True),
             skip_existing=gen.get("skip_existing", True),
             login_required=site_cfg.get("login_required", False),
             save_html=debug.get("save_html", False),

novel_downloader/config/models.py

@@ -35,6 +35,7 @@ class RequesterConfig:
     disable_images: bool = True
     mute_audio: bool = True
     mode: str = "session"  # browser / session / async
+    max_rps: Optional[float] = None  # Maximum requests per second
 
 
 # === Downloaders ===
@@ -43,7 +44,9 @@ class DownloaderConfig:
     request_interval: int = 5
     raw_data_dir: str = "./raw_data"
     cache_dir: str = "./novel_cache"
-    max_threads: int = 4
+    download_workers: int = 4
+    parser_workers: int = 4
+    use_process_pool: bool = False
     skip_existing: bool = True
     login_required: bool = False
     save_html: bool = False

novel_downloader/core/downloaders/__init__.py

@@ -11,10 +11,12 @@ Each downloader is responsible for orchestrating the full lifecycle
 of retrieving, parsing, and saving novel content for a given source.
 """
 
+from .common_asynb_downloader import CommonAsyncDownloader
 from .common_downloader import CommonDownloader
 from .qidian_downloader import QidianDownloader
 
 __all__ = [
+    "CommonAsyncDownloader",
     "CommonDownloader",
     "QidianDownloader",
 ]

novel_downloader/core/downloaders/base_async_downloader.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.base_async_downloader
+-------------------------------------------------------
+
+Defines the abstract base class `BaseAsyncDownloader`, which provides a
+common interface and reusable logic for all downloader implementations.
+"""
+
+import abc
+import logging
+from pathlib import Path
+from typing import List
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+from novel_downloader.core.interfaces.async_downloader_protocol import (
+    AsyncDownloaderProtocol,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
+    """
+    Abstract downloader that defines the initialization interface
+    and the general batch download flow.
+
+    Subclasses must implement the logic for downloading a single book.
+    """
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        self._requester = requester
+        self._parser = parser
+        self._saver = saver
+        self._config = config
+        self._site = site
+
+        self._raw_data_dir = Path(config.raw_data_dir) / site
+        self._cache_dir = Path(config.cache_dir) / site
+        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
+        self._cache_dir.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def requester(self) -> AsyncRequesterProtocol:
+        return self._requester
+
+    @property
+    def parser(self) -> ParserProtocol:
+        return self._parser
+
+    @property
+    def saver(self) -> SaverProtocol:
+        return self._saver
+
+    @property
+    def config(self) -> DownloaderConfig:
+        return self._config
+
+    @property
+    def raw_data_dir(self) -> Path:
+        return self._raw_data_dir
+
+    @property
+    def cache_dir(self) -> Path:
+        return self._cache_dir
+
+    @property
+    def site(self) -> str:
+        return self._site
+
+    @property
+    def save_html(self) -> bool:
+        return self._config.save_html
+
+    @property
+    def skip_existing(self) -> bool:
+        return self._config.skip_existing
+
+    @property
+    def login_required(self) -> bool:
+        return self._config.login_required
+
+    @property
+    def request_interval(self) -> int:
+        return self._config.request_interval
+
+    async def prepare(self) -> None:
+        """
+        Optional hook called before downloading each book.
+
+        Subclasses can override this method to perform pre-download setup.
+        """
+        return
+
+    async def download(self, book_ids: List[str]) -> None:
+        """
+        The general batch download process:
+          1. Iterate over all book IDs
+          2. For each ID, call `download_one()`
+
+        :param book_ids: A list of book identifiers to download.
+        """
+        await self.prepare()
+
+        # 2) batch download
+        for idx, book_id in enumerate(book_ids, start=1):
+            logger.debug(
+                "[%s] Starting download for %r (%s/%s)",
+                self.__class__.__name__,
+                book_id,
+                idx,
+                len(book_ids),
+            )
+            try:
+                await self.download_one(book_id)
+            except Exception as e:
+                self._handle_download_exception(book_id, e)
+
+    @abc.abstractmethod
+    async def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        Subclasses must implement this method.
+
+        :param book_id: The identifier of the book to download.
+        """
+        ...
+
+    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+        """
+        Handle download errors in a consistent way.
+
+        This method can be overridden or extended to implement retry logic, etc.
+
+        :param book_id: The ID of the book that failed.
+        :param error: The exception raised during download.
+        """
+        logger.warning(
+            "[%s] Failed to download %r: %s",
+            self.__class__.__name__,
+            book_id,
+            error,
+        )

novel_downloader/core/downloaders/common_asynb_downloader.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.common_asynb_downloader
+---------------------------------------------------------
+
+This module defines `CommonAsynbDownloader`.
+"""
+
+import asyncio
+import json
+import logging
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from typing import Any, Dict, Tuple
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_async_downloader import BaseAsyncDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class CommonAsyncDownloader(BaseAsyncDownloader):
+    """
+    Specialized Async downloader for common novels.
+    """
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        """ """
+        super().__init__(requester, parser, saver, config, site)
+        self._is_logged_in = False
+
+    async def prepare(self) -> None:
+        """
+        Perform login
+        """
+        if self.login_required and not self._is_logged_in:
+            success = await self.requester.login(max_retries=3)
+            if not success:
+                raise RuntimeError("Login failed")
+            self._is_logged_in = True
+
+    async def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        assert isinstance(self.requester, AsyncRequesterProtocol)
+
+        TAG = "[AsyncDownloader]"
+        raw_base = self.raw_data_dir / book_id
+        cache_base = self.cache_dir / book_id
+        info_path = raw_base / "book_info.json"
+        chapters_html_dir = cache_base / "html"
+        chapter_dir = raw_base / "chapters"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+        if self.save_html:
+            chapters_html_dir.mkdir(parents=True, exist_ok=True)
+
+        # load or fetch book_info
+        book_info: Dict[str, Any]
+        re_fetch = True
+        if info_path.exists():
+            try:
+                data = json.loads(info_path.read_text("utf-8"))
+                days, *_ = calculate_time_difference(
+                    data.get("update_time", ""), "UTC+8"
+                )
+                re_fetch = days > 1
+            except Exception:
+                re_fetch = True
+
+        if re_fetch:
+            info_html = await self.requester.get_book_info(
+                book_id, self.request_interval
+            )
+            if self.save_html:
+                save_as_txt(info_html, chapters_html_dir / "info.html")
+            book_info = self.parser.parse_book_info(info_html)
+            if book_info.get("book_name") != "未找到书名":
+                save_as_json(book_info, info_path)
+            else:
+                logger.warning("%s 书籍信息未找到, book_id = %s", TAG, book_id)
+        else:
+            book_info = json.loads(info_path.read_text("utf-8"))
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            await asyncio.get_running_loop().run_in_executor(
+                None, download_image_as_bytes, cover_url, raw_base
+            )
+
+        # setup queue, semaphore, executor
+        semaphore = asyncio.Semaphore(self.download_workers)
+        queue: asyncio.Queue[Tuple[str, str]] = asyncio.Queue()
+        loop = asyncio.get_running_loop()
+        executor = (
+            ProcessPoolExecutor() if self.use_process_pool else ThreadPoolExecutor()
+        )
+
+        async def parser_worker(worker_id: int) -> None:
+            while True:
+                cid, html = await queue.get()
+                try:
+                    chap_json = await loop.run_in_executor(
+                        executor, self.parser.parse_chapter, html, cid
+                    )
+                    if chap_json:
+                        await loop.run_in_executor(
+                            executor,
+                            save_as_json,
+                            chap_json,
+                            chapter_dir / f"{cid}.json",
+                        )
+                        logger.info(
+                            "%s [Parser-%d] saved chapter %s", TAG, worker_id, cid
+                        )
+                except Exception as e:
+                    logger.error(
+                        "%s [Parser-%d] error on chapter %s: %s", TAG, worker_id, cid, e
+                    )
+                finally:
+                    queue.task_done()
+
+        async def download_worker(chap: Dict[str, Any]) -> None:
+            cid = str(chap.get("chapterId") or "")
+            if not cid:
+                return
+            target = chapter_dir / f"{cid}.json"
+            if target.exists() and self.skip_existing:
+                logger.info("%s skipping existing chapter %s", TAG, cid)
+                return
+
+            try:
+                async with semaphore:
+                    html = await self.requester.get_book_chapter(
+                        book_id, cid, self.request_interval
+                    )
+                if self.save_html:
+                    await loop.run_in_executor(
+                        executor,
+                        save_as_txt,
+                        html,
+                        chapters_html_dir / f"{cid}.html",
+                    )
+                await queue.put((cid, html))
+                logger.info("%s downloaded chapter %s", TAG, cid)
+            except Exception as e:
+                logger.error("%s error downloading %s: %s", TAG, cid, e)
+
+        # start parser workers
+        parsers = [
+            asyncio.create_task(parser_worker(i)) for i in range(self.parser_workers)
+        ]
+
+        # enqueue + run downloads
+        download_tasks = []
+        for vol in book_info.get("volumes", []):
+            for chap in vol.get("chapters", []):
+                download_tasks.append(asyncio.create_task(download_worker(chap)))
+
+        await asyncio.gather(*download_tasks)
+        await queue.join()  # wait until all parsed
+        for p in parsers:
+            p.cancel()  # stop parser loops
+
+        # final save
+        await loop.run_in_executor(executor, self.saver.save, book_id)
+        executor.shutdown(wait=True)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+    @property
+    def parser_workers(self) -> int:
+        return self.config.parser_workers
+
+    @property
+    def download_workers(self) -> int:
+        return self.config.download_workers
+
+    @property
+    def use_process_pool(self) -> bool:
+        return self.config.use_process_pool

novel_downloader/core/downloaders/common_downloader.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.qidian_downloader
+novel_downloader.core.downloaders.common_downloader
 ---------------------------------------------------
 
-This module defines `QidianDownloader`, a platform-specific downloader
-implementation for retrieving novels from Qidian (起点中文网).
+This module defines `CommonDownloader`.
 """
 
 import json

novel_downloader/core/factory/__init__.py

@@ -8,14 +8,26 @@ This package provides factory methods for dynamically retrieving components
 based on runtime parameters such as site name or content type.
 """
 
-from .downloader_factory import get_downloader
+from .downloader_factory import (
+    get_async_downloader,
+    get_downloader,
+    get_sync_downloader,
+)
 from .parser_factory import get_parser
-from .requester_factory import get_requester
+from .requester_factory import (
+    get_async_requester,
+    get_requester,
+    get_sync_requester,
+)
 from .saver_factory import get_saver
 
 __all__ = [
+    "get_async_downloader",
     "get_downloader",
+    "get_sync_downloader",
     "get_parser",
+    "get_async_requester",
     "get_requester",
+    "get_sync_requester",
     "get_saver",
 ]