novel-downloader 1.3.3__py3-none-any.whl → 1.4.1__py3-none-any.whl

novel_downloader/core/downloaders/base/base_async.py

@@ -1,153 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.base.base_async
--------------------------------------------------
-
-Defines the abstract base class `BaseAsyncDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
-"""
-
-import abc
-import logging
-from pathlib import Path
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.interfaces import (
-    AsyncDownloaderProtocol,
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
-    """
-    Abstract downloader that defines the initialization interface
-    and the general batch download flow.
-
-    Subclasses must implement the logic for downloading a single book.
-    """
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        self._requester = requester
-        self._parser = parser
-        self._saver = saver
-        self._config = config
-        self._site = site
-
-        self._raw_data_dir = Path(config.raw_data_dir) / site
-        self._cache_dir = Path(config.cache_dir) / site
-        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
-
-        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
-
-    async def download(self, book_ids: list[str]) -> None:
-        """
-        The general batch download process:
-          1. Iterate over all book IDs
-          2. For each ID, call `download_one()`
-
-        :param book_ids: A list of book identifiers to download.
-        """
-        await self.prepare()
-
-        # 2) batch download
-        for idx, book_id in enumerate(book_ids, start=1):
-            self.logger.debug(
-                "[%s] Starting download for %r (%s/%s)",
-                self.__class__.__name__,
-                book_id,
-                idx,
-                len(book_ids),
-            )
-            try:
-                await self.download_one(book_id)
-            except Exception as e:
-                self._handle_download_exception(book_id, e)
-
-    @abc.abstractmethod
-    async def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
-    async def prepare(self) -> None:
-        """
-        Optional hook called before downloading each book.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    @property
-    def requester(self) -> AsyncRequesterProtocol:
-        return self._requester
-
-    @property
-    def parser(self) -> ParserProtocol:
-        return self._parser
-
-    @property
-    def saver(self) -> SaverProtocol:
-        return self._saver
-
-    @property
-    def config(self) -> DownloaderConfig:
-        return self._config
-
-    @property
-    def raw_data_dir(self) -> Path:
-        return self._raw_data_dir
-
-    @property
-    def cache_dir(self) -> Path:
-        return self._cache_dir
-
-    @property
-    def site(self) -> str:
-        return self._site
-
-    @property
-    def save_html(self) -> bool:
-        return self._config.save_html
-
-    @property
-    def skip_existing(self) -> bool:
-        return self._config.skip_existing
-
-    @property
-    def login_required(self) -> bool:
-        return self._config.login_required
-
-    @property
-    def request_interval(self) -> float:
-        return self._config.request_interval
-
-    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
-        """
-        Handle download errors in a consistent way.
-
-        This method can be overridden or extended to implement retry logic, etc.
-
-        :param book_id: The ID of the book that failed.
-        :param error: The exception raised during download.
-        """
-        self.logger.warning(
-            "[%s] Failed to download %r: %s",
-            self.__class__.__name__,
-            book_id,
-            error,
-        )

novel_downloader/core/downloaders/base/base_sync.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.base.base_sync
-------------------------------------------------
-
-Defines the abstract base class `BaseDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
-"""
-
-import abc
-import logging
-from pathlib import Path
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncDownloaderProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class BaseDownloader(SyncDownloaderProtocol, abc.ABC):
-    """
-    Abstract downloader that defines the initialization interface
-    and the general batch download flow.
-
-    Subclasses must implement the logic for downloading a single book.
-    """
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        """
-        Initialize the downloader with its components.
-
-        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
-        :param parser: Object implementing ParserProtocol, used to parse page content.
-        :param saver: Object implementing SaverProtocol, used to save final output.
-        :param config: Downloader configuration object.
-        """
-        self._requester = requester
-        self._parser = parser
-        self._saver = saver
-        self._config = config
-        self._site = site
-
-        self._raw_data_dir = Path(config.raw_data_dir) / site
-        self._cache_dir = Path(config.cache_dir) / site
-        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
-
-        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
-
-    def download(self, book_ids: list[str]) -> None:
-        """
-        The general batch download process:
-          1. Iterate over all book IDs
-          2. For each ID, call `download_one()`
-
-        :param book_ids: A list of book identifiers to download.
-        """
-        self.prepare()
-
-        for idx, book_id in enumerate(book_ids, start=1):
-            self.logger.debug(
-                "[downloader] Starting download for book_id: %s (%s/%s)",
-                book_id,
-                idx,
-                len(book_ids),
-            )
-            try:
-                self.download_one(book_id)
-            except Exception as e:
-                self._handle_download_exception(book_id, e)
-
-    @abc.abstractmethod
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
-    def prepare(self) -> None:
-        """
-        Optional hook called before downloading each book.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    @property
-    def requester(self) -> SyncRequesterProtocol:
-        """
-        Access the current requester.
-
-        :return: The internal requester instance.
-        """
-        return self._requester
-
-    @property
-    def parser(self) -> ParserProtocol:
-        """
-        Access the current parser.
-
-        :return: The internal parser instance.
-        """
-        return self._parser
-
-    @property
-    def saver(self) -> SaverProtocol:
-        """
-        Access the current saver.
-
-        :return: The internal saver instance.
-        """
-        return self._saver
-
-    @property
-    def config(self) -> DownloaderConfig:
-        """
-        Access the downloader configuration.
-
-        :return: The internal DownloaderConfig object.
-        """
-        return self._config
-
-    @property
-    def raw_data_dir(self) -> Path:
-        """
-        Access the root directory for storing raw downloaded data.
-
-        :return: Path to the raw data directory.
-        """
-        return self._raw_data_dir
-
-    @property
-    def cache_dir(self) -> Path:
-        """
-        Access the directory used for temporary caching during download.
-
-        :return: Path to the cache directory.
-        """
-        return self._cache_dir
-
-    @property
-    def site(self) -> str:
-        return self._site
-
-    @property
-    def save_html(self) -> bool:
-        return self._config.save_html
-
-    @property
-    def skip_existing(self) -> bool:
-        return self._config.skip_existing
-
-    @property
-    def login_required(self) -> bool:
-        return self._config.login_required
-
-    @property
-    def request_interval(self) -> float:
-        return self._config.request_interval
-
-    def set_requester(self, requester: SyncRequesterProtocol) -> None:
-        """
-        Replace the requester instance with a new one.
-
-        :param requester: The new requester to be used.
-        """
-        self._requester = requester
-
-    def set_parser(self, parser: ParserProtocol) -> None:
-        """
-        Replace the parser instance with a new one.
-
-        :param parser: The new parser to be used.
-        """
-        self._parser = parser
-
-    def set_saver(self, saver: SaverProtocol) -> None:
-        """
-        Replace the saver instance with a new one.
-
-        :param saver: The new saver to be used.
-        """
-        self._saver = saver
-
-    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
-        """
-        Handle download errors in a consistent way.
-
-        This method can be overridden or extended to implement retry logic, etc.
-
-        :param book_id: The ID of the book that failed.
-        :param error: The exception raised during download.
-        """
-        self.logger.warning("[downloader] Failed to download %s: %s", book_id, error)

novel_downloader/core/downloaders/biquge/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge
-----------------------------------------
-
-"""
-
-from .biquge_async import BiqugeAsyncDownloader
-from .biquge_sync import BiqugeDownloader
-
-__all__ = [
-    "BiqugeAsyncDownloader",
-    "BiqugeDownloader",
-]

novel_downloader/core/downloaders/biquge/biquge_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge.biquge_async
------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class BiqugeAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/biquge/biquge_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge.biquge_sync
-----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class BiqugeDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/common/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.common
-----------------------------------------
-
-"""
-
-from .common_async import CommonAsyncDownloader
-from .common_sync import CommonDownloader
-
-__all__ = [
-    "CommonAsyncDownloader",
-    "CommonDownloader",
-]

novel_downloader/core/downloaders/common/common_async.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.common.common_async
------------------------------------------------------
-
-"""
-
-import asyncio
-import json
-import logging
-from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
-from typing import Any
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.downloaders.base import BaseAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-from novel_downloader.utils.chapter_storage import ChapterDict, ChapterStorage
-from novel_downloader.utils.file_utils import save_as_json, save_as_txt
-from novel_downloader.utils.time_utils import calculate_time_difference
-
-logger = logging.getLogger(__name__)
-
-
-class CommonAsyncDownloader(BaseAsyncDownloader):
-    """
-    Specialized Async downloader for common novels.
-    """
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        """ """
-        super().__init__(requester, parser, saver, config, site)
-        self._is_logged_in = False
-
-    async def prepare(self) -> None:
-        """
-        Perform login
-        """
-        if self.login_required and not self._is_logged_in:
-            success = await self.requester.login()
-            if not success:
-                raise RuntimeError("Login failed")
-            self._is_logged_in = True
-
-    async def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        :param book_id: The identifier of the book to download.
-        """
-        assert isinstance(self.requester, AsyncRequesterProtocol)
-        await self.prepare()
-
-        TAG = "[AsyncDownloader]"
-        wait_time = self.config.request_interval
-
-        raw_base = self.raw_data_dir / book_id
-        cache_base = self.cache_dir / book_id
-        info_path = raw_base / "book_info.json"
-        chapters_html_dir = cache_base / "html"
-
-        raw_base.mkdir(parents=True, exist_ok=True)
-        if self.save_html:
-            chapters_html_dir.mkdir(parents=True, exist_ok=True)
-        normal_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-
-        # load or fetch book_info
-        book_info: dict[str, Any]
-        re_fetch = True
-        if info_path.exists():
-            try:
-                data = json.loads(info_path.read_text("utf-8"))
-                days, *_ = calculate_time_difference(
-                    data.get("update_time", ""), "UTC+8"
-                )
-                re_fetch = days > 1
-            except Exception:
-                re_fetch = True
-
-        if re_fetch:
-            info_html = await self.requester.get_book_info(book_id)
-            if self.save_html:
-                for i, html in enumerate(info_html):
-                    save_as_txt(html, chapters_html_dir / f"info_{i}.html")
-            book_info = self.parser.parse_book_info(info_html)
-            if book_info.get("book_name") != "未找到书名":
-                save_as_json(book_info, info_path)
-            else:
-                logger.warning("%s 书籍信息未找到, book_id = %s", TAG, book_id)
-            await asyncio.sleep(wait_time)
-        else:
-            book_info = json.loads(info_path.read_text("utf-8"))
-
-        # setup queue, semaphore, executor
-        semaphore = asyncio.Semaphore(self.download_workers)
-        queue: asyncio.Queue[tuple[str, list[str]]] = asyncio.Queue()
-        save_queue: asyncio.Queue[ChapterDict] = asyncio.Queue()
-        loop = asyncio.get_running_loop()
-        executor = (
-            ProcessPoolExecutor() if self.use_process_pool else ThreadPoolExecutor()
-        )
-
-        async def parser_worker(worker_id: int) -> None:
-            while True:
-                cid, html = await queue.get()
-                try:
-                    chap_json = await loop.run_in_executor(
-                        executor, self.parser.parse_chapter, html, cid
-                    )
-                    if chap_json:
-                        await save_queue.put(chap_json)
-                        logger.info(
-                            "%s [Parser-%d] saved chapter %s", TAG, worker_id, cid
-                        )
-                except Exception as e:
-                    logger.error(
-                        "%s [Parser-%d] error on chapter %s: %s", TAG, worker_id, cid, e
-                    )
-                finally:
-                    queue.task_done()
-
-        async def saver_loop(
-            cs: ChapterStorage,
-            queue: asyncio.Queue[ChapterDict],
-        ) -> None:
-            while True:
-                data = await queue.get()
-                try:
-                    cs.save(data)
-                except Exception as e:
-                    logger.error(
-                        "[saver] Error saving chapter %s: %s",
-                        data.get("id"),
-                        e,
-                    )
-                finally:
-                    queue.task_done()
-
-        async def download_worker(chap: dict[str, Any]) -> None:
-            cid = str(chap.get("chapterId") or "")
-            if not cid:
-                return
-            if normal_cs.exists(cid) and self.skip_existing:
-                logger.info("%s skipping existing chapter %s", TAG, cid)
-                return
-
-            try:
-                async with semaphore:
-                    html = await self.requester.get_book_chapter(book_id, cid)
-                await queue.put((cid, html))
-                logger.info("%s downloaded chapter %s", TAG, cid)
-            except Exception as e:
-                logger.error("%s error downloading %s: %s", TAG, cid, e)
-
-        # start parser workers
-        parsers = [
-            asyncio.create_task(parser_worker(i)) for i in range(self.parser_workers)
-        ]
-        chapter_saver = asyncio.create_task(saver_loop(normal_cs, save_queue))
-
-        # enqueue + run downloads
-        download_tasks = []
-        for vol in book_info.get("volumes", []):
-            for chap in vol.get("chapters", []):
-                download_tasks.append(asyncio.create_task(download_worker(chap)))
-
-        await asyncio.gather(*download_tasks)
-        await queue.join()  # wait until all parsed
-        await save_queue.join()
-        for p in parsers:
-            p.cancel()  # stop parser loops
-        chapter_saver.cancel()
-
-        # final save
-        await loop.run_in_executor(executor, self.saver.save, book_id)
-        executor.shutdown(wait=True)
-
-        logger.info(
-            "%s Novel '%s' download completed.",
-            TAG,
-            book_info.get("book_name", "unknown"),
-        )
-        return
-
-    @property
-    def parser_workers(self) -> int:
-        return self.config.parser_workers
-
-    @property
-    def download_workers(self) -> int:
-        return self.config.download_workers
-
-    @property
-    def use_process_pool(self) -> bool:
-        return self.config.use_process_pool