novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/core/downloaders/base_async_downloader.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.base_async_downloader
+-------------------------------------------------------
+
+Defines the abstract base class `BaseAsyncDownloader`, which provides a
+common interface and reusable logic for all downloader implementations.
+"""
+
+import abc
+import logging
+from pathlib import Path
+from typing import List
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+from novel_downloader.core.interfaces.async_downloader_protocol import (
+    AsyncDownloaderProtocol,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
+    """
+    Abstract downloader that defines the initialization interface
+    and the general batch download flow.
+
+    Subclasses must implement the logic for downloading a single book.
+    """
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        self._requester = requester
+        self._parser = parser
+        self._saver = saver
+        self._config = config
+        self._site = site
+
+        self._raw_data_dir = Path(config.raw_data_dir) / site
+        self._cache_dir = Path(config.cache_dir) / site
+        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
+        self._cache_dir.mkdir(parents=True, exist_ok=True)
+
+    @property
+    def requester(self) -> AsyncRequesterProtocol:
+        return self._requester
+
+    @property
+    def parser(self) -> ParserProtocol:
+        return self._parser
+
+    @property
+    def saver(self) -> SaverProtocol:
+        return self._saver
+
+    @property
+    def config(self) -> DownloaderConfig:
+        return self._config
+
+    @property
+    def raw_data_dir(self) -> Path:
+        return self._raw_data_dir
+
+    @property
+    def cache_dir(self) -> Path:
+        return self._cache_dir
+
+    @property
+    def site(self) -> str:
+        return self._site
+
+    @property
+    def save_html(self) -> bool:
+        return self._config.save_html
+
+    @property
+    def skip_existing(self) -> bool:
+        return self._config.skip_existing
+
+    @property
+    def login_required(self) -> bool:
+        return self._config.login_required
+
+    @property
+    def request_interval(self) -> int:
+        return self._config.request_interval
+
+    async def prepare(self) -> None:
+        """
+        Optional hook called before downloading each book.
+
+        Subclasses can override this method to perform pre-download setup.
+        """
+        return
+
+    async def download(self, book_ids: List[str]) -> None:
+        """
+        The general batch download process:
+          1. Iterate over all book IDs
+          2. For each ID, call `download_one()`
+
+        :param book_ids: A list of book identifiers to download.
+        """
+        await self.prepare()
+
+        # 2) batch download
+        for idx, book_id in enumerate(book_ids, start=1):
+            logger.debug(
+                "[%s] Starting download for %r (%s/%s)",
+                self.__class__.__name__,
+                book_id,
+                idx,
+                len(book_ids),
+            )
+            try:
+                await self.download_one(book_id)
+            except Exception as e:
+                self._handle_download_exception(book_id, e)
+
+    @abc.abstractmethod
+    async def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        Subclasses must implement this method.
+
+        :param book_id: The identifier of the book to download.
+        """
+        ...
+
+    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+        """
+        Handle download errors in a consistent way.
+
+        This method can be overridden or extended to implement retry logic, etc.
+
+        :param book_id: The ID of the book that failed.
+        :param error: The exception raised during download.
+        """
+        logger.warning(
+            "[%s] Failed to download %r: %s",
+            self.__class__.__name__,
+            book_id,
+            error,
+        )

novel_downloader/core/downloaders/base_downloader.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.base_downloader
+-------------------------------------------------
+
+Defines the abstract base class `BaseDownloader`, which provides a
+common interface and reusable logic for all downloader implementations.
+"""
+
+import abc
+import logging
+from pathlib import Path
+from typing import List
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    DownloaderProtocol,
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseDownloader(DownloaderProtocol, abc.ABC):
+    """
+    Abstract downloader that defines the initialization interface
+    and the general batch download flow.
+
+    Subclasses must implement the logic for downloading a single book.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        """
+        Initialize the downloader with its components.
+
+        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
+        :param parser: Object implementing ParserProtocol, used to parse page content.
+        :param saver: Object implementing SaverProtocol, used to save final output.
+        :param config: Downloader configuration object.
+        """
+        self._requester = requester
+        self._parser = parser
+        self._saver = saver
+        self._config = config
+        self._raw_data_dir = Path(config.raw_data_dir)
+        self._cache_dir = Path(config.cache_dir)
+        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
+        self._cache_dir.mkdir(parents=True, exist_ok=True)
+
+    @abc.abstractmethod
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        Subclasses must implement this method.
+
+        :param book_id: The identifier of the book to download.
+        """
+        ...
+
+    def download(self, book_ids: List[str]) -> None:
+        """
+        The general batch download process:
+          1. Iterate over all book IDs
+          2. For each ID, call `download_one()`
+
+        :param book_ids: A list of book identifiers to download.
+        """
+        for idx, book_id in enumerate(book_ids, start=1):
+            try:
+                logger.debug(
+                    "[downloader] Starting download for book_id: %s (%s/%s)",
+                    book_id,
+                    idx,
+                    len(book_ids),
+                )
+                self.download_one(book_id)
+            except Exception as e:
+                self._handle_download_exception(book_id, e)
+
+    def before_download(self, book_id: str) -> None:
+        """
+        Optional hook called before downloading each book.
+
+        Subclasses can override this method to perform pre-download setup.
+
+        :param book_id: The book ID about to be processed.
+        """
+        pass
+
+    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+        """
+        Handle download errors in a consistent way.
+
+        This method can be overridden or extended to implement retry logic, etc.
+
+        :param book_id: The ID of the book that failed.
+        :param error: The exception raised during download.
+        """
+        logger.warning("[downloader] Failed to download %s: %s", book_id, error)
+
+    @property
+    def requester(self) -> RequesterProtocol:
+        """
+        Access the current requester.
+
+        :return: The internal requester instance.
+        """
+        return self._requester
+
+    @property
+    def parser(self) -> ParserProtocol:
+        """
+        Access the current parser.
+
+        :return: The internal parser instance.
+        """
+        return self._parser
+
+    @property
+    def saver(self) -> SaverProtocol:
+        """
+        Access the current saver.
+
+        :return: The internal saver instance.
+        """
+        return self._saver
+
+    @property
+    def config(self) -> DownloaderConfig:
+        """
+        Access the downloader configuration.
+
+        :return: The internal DownloaderConfig object.
+        """
+        return self._config
+
+    @property
+    def raw_data_dir(self) -> Path:
+        """
+        Access the root directory for storing raw downloaded data.
+
+        :return: Path to the raw data directory.
+        """
+        return self._raw_data_dir
+
+    @property
+    def cache_dir(self) -> Path:
+        """
+        Access the directory used for temporary caching during download.
+
+        :return: Path to the cache directory.
+        """
+        return self._cache_dir
+
+    def set_requester(self, requester: RequesterProtocol) -> None:
+        """
+        Replace the requester instance with a new one.
+
+        :param requester: The new requester to be used.
+        """
+        self._requester = requester
+
+    def set_parser(self, parser: ParserProtocol) -> None:
+        """
+        Replace the parser instance with a new one.
+
+        :param parser: The new parser to be used.
+        """
+        self._parser = parser
+
+    def set_saver(self, saver: SaverProtocol) -> None:
+        """
+        Replace the saver instance with a new one.
+
+        :param saver: The new saver to be used.
+        """
+        self._saver = saver

novel_downloader/core/downloaders/common_asynb_downloader.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.common_asynb_downloader
+---------------------------------------------------------
+
+This module defines `CommonAsynbDownloader`.
+"""
+
+import asyncio
+import json
+import logging
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from typing import Any, Dict, Tuple
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    ParserProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_async_downloader import BaseAsyncDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class CommonAsyncDownloader(BaseAsyncDownloader):
+    """
+    Specialized Async downloader for common novels.
+    """
+
+    def __init__(
+        self,
+        requester: AsyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        """ """
+        super().__init__(requester, parser, saver, config, site)
+        self._is_logged_in = False
+
+    async def prepare(self) -> None:
+        """
+        Perform login
+        """
+        if self.login_required and not self._is_logged_in:
+            success = await self.requester.login(max_retries=3)
+            if not success:
+                raise RuntimeError("Login failed")
+            self._is_logged_in = True
+
+    async def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        assert isinstance(self.requester, AsyncRequesterProtocol)
+
+        TAG = "[AsyncDownloader]"
+        raw_base = self.raw_data_dir / book_id
+        cache_base = self.cache_dir / book_id
+        info_path = raw_base / "book_info.json"
+        chapters_html_dir = cache_base / "html"
+        chapter_dir = raw_base / "chapters"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+        if self.save_html:
+            chapters_html_dir.mkdir(parents=True, exist_ok=True)
+
+        # load or fetch book_info
+        book_info: Dict[str, Any]
+        re_fetch = True
+        if info_path.exists():
+            try:
+                data = json.loads(info_path.read_text("utf-8"))
+                days, *_ = calculate_time_difference(
+                    data.get("update_time", ""), "UTC+8"
+                )
+                re_fetch = days > 1
+            except Exception:
+                re_fetch = True
+
+        if re_fetch:
+            info_html = await self.requester.get_book_info(
+                book_id, self.request_interval
+            )
+            if self.save_html:
+                save_as_txt(info_html, chapters_html_dir / "info.html")
+            book_info = self.parser.parse_book_info(info_html)
+            if book_info.get("book_name") != "未找到书名":
+                save_as_json(book_info, info_path)
+            else:
+                logger.warning("%s 书籍信息未找到, book_id = %s", TAG, book_id)
+        else:
+            book_info = json.loads(info_path.read_text("utf-8"))
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            await asyncio.get_running_loop().run_in_executor(
+                None, download_image_as_bytes, cover_url, raw_base
+            )
+
+        # setup queue, semaphore, executor
+        semaphore = asyncio.Semaphore(self.download_workers)
+        queue: asyncio.Queue[Tuple[str, str]] = asyncio.Queue()
+        loop = asyncio.get_running_loop()
+        executor = (
+            ProcessPoolExecutor() if self.use_process_pool else ThreadPoolExecutor()
+        )
+
+        async def parser_worker(worker_id: int) -> None:
+            while True:
+                cid, html = await queue.get()
+                try:
+                    chap_json = await loop.run_in_executor(
+                        executor, self.parser.parse_chapter, html, cid
+                    )
+                    if chap_json:
+                        await loop.run_in_executor(
+                            executor,
+                            save_as_json,
+                            chap_json,
+                            chapter_dir / f"{cid}.json",
+                        )
+                        logger.info(
+                            "%s [Parser-%d] saved chapter %s", TAG, worker_id, cid
+                        )
+                except Exception as e:
+                    logger.error(
+                        "%s [Parser-%d] error on chapter %s: %s", TAG, worker_id, cid, e
+                    )
+                finally:
+                    queue.task_done()
+
+        async def download_worker(chap: Dict[str, Any]) -> None:
+            cid = str(chap.get("chapterId") or "")
+            if not cid:
+                return
+            target = chapter_dir / f"{cid}.json"
+            if target.exists() and self.skip_existing:
+                logger.info("%s skipping existing chapter %s", TAG, cid)
+                return
+
+            try:
+                async with semaphore:
+                    html = await self.requester.get_book_chapter(
+                        book_id, cid, self.request_interval
+                    )
+                if self.save_html:
+                    await loop.run_in_executor(
+                        executor,
+                        save_as_txt,
+                        html,
+                        chapters_html_dir / f"{cid}.html",
+                    )
+                await queue.put((cid, html))
+                logger.info("%s downloaded chapter %s", TAG, cid)
+            except Exception as e:
+                logger.error("%s error downloading %s: %s", TAG, cid, e)
+
+        # start parser workers
+        parsers = [
+            asyncio.create_task(parser_worker(i)) for i in range(self.parser_workers)
+        ]
+
+        # enqueue + run downloads
+        download_tasks = []
+        for vol in book_info.get("volumes", []):
+            for chap in vol.get("chapters", []):
+                download_tasks.append(asyncio.create_task(download_worker(chap)))
+
+        await asyncio.gather(*download_tasks)
+        await queue.join()  # wait until all parsed
+        for p in parsers:
+            p.cancel()  # stop parser loops
+
+        # final save
+        await loop.run_in_executor(executor, self.saver.save, book_id)
+        executor.shutdown(wait=True)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+    @property
+    def parser_workers(self) -> int:
+        return self.config.parser_workers
+
+    @property
+    def download_workers(self) -> int:
+        return self.config.download_workers
+
+    @property
+    def use_process_pool(self) -> bool:
+        return self.config.use_process_pool