novel-downloader 1.1.1__py3-none-any.whl

novel_downloader/core/downloaders/base_downloader.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.base_downloader
+-------------------------------------------------
+
+Defines the abstract base class `BaseDownloader`, which provides a
+common interface and reusable logic for all downloader implementations.
+"""
+
+import abc
+import logging
+from pathlib import Path
+from typing import List
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    DownloaderProtocol,
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseDownloader(DownloaderProtocol, abc.ABC):
+    """
+    Abstract downloader that defines the initialization interface
+    and the general batch download flow.
+
+    Subclasses must implement the logic for downloading a single book.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        """
+        Initialize the downloader with its components.
+
+        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
+        :param parser: Object implementing ParserProtocol, used to parse page content.
+        :param saver: Object implementing SaverProtocol, used to save final output.
+        :param config: Downloader configuration object.
+        """
+        self._requester = requester
+        self._parser = parser
+        self._saver = saver
+        self._config = config
+        self._raw_data_dir = Path(config.raw_data_dir)
+        self._cache_dir = Path(config.cache_dir)
+        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
+        self._cache_dir.mkdir(parents=True, exist_ok=True)
+
+    @abc.abstractmethod
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        Subclasses must implement this method.
+
+        :param book_id: The identifier of the book to download.
+        """
+        ...
+
+    def download(self, book_ids: List[str]) -> None:
+        """
+        The general batch download process:
+          1. Iterate over all book IDs
+          2. For each ID, call `download_one()`
+
+        :param book_ids: A list of book identifiers to download.
+        """
+        for idx, book_id in enumerate(book_ids, start=1):
+            try:
+                logger.debug(
+                    "[downloader] Starting download for book_id: %s (%s/%s)",
+                    book_id,
+                    idx,
+                    len(book_ids),
+                )
+                self.download_one(book_id)
+            except Exception as e:
+                self._handle_download_exception(book_id, e)
+
+    def before_download(self, book_id: str) -> None:
+        """
+        Optional hook called before downloading each book.
+
+        Subclasses can override this method to perform pre-download setup.
+
+        :param book_id: The book ID about to be processed.
+        """
+        pass
+
+    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+        """
+        Handle download errors in a consistent way.
+
+        This method can be overridden or extended to implement retry logic, etc.
+
+        :param book_id: The ID of the book that failed.
+        :param error: The exception raised during download.
+        """
+        logger.warning("[downloader] Failed to download %s: %s", book_id, error)
+
+    @property
+    def requester(self) -> RequesterProtocol:
+        """
+        Access the current requester.
+
+        :return: The internal requester instance.
+        """
+        return self._requester
+
+    @property
+    def parser(self) -> ParserProtocol:
+        """
+        Access the current parser.
+
+        :return: The internal parser instance.
+        """
+        return self._parser
+
+    @property
+    def saver(self) -> SaverProtocol:
+        """
+        Access the current saver.
+
+        :return: The internal saver instance.
+        """
+        return self._saver
+
+    @property
+    def config(self) -> DownloaderConfig:
+        """
+        Access the downloader configuration.
+
+        :return: The internal DownloaderConfig object.
+        """
+        return self._config
+
+    @property
+    def raw_data_dir(self) -> Path:
+        """
+        Access the root directory for storing raw downloaded data.
+
+        :return: Path to the raw data directory.
+        """
+        return self._raw_data_dir
+
+    @property
+    def cache_dir(self) -> Path:
+        """
+        Access the directory used for temporary caching during download.
+
+        :return: Path to the cache directory.
+        """
+        return self._cache_dir
+
+    def set_requester(self, requester: RequesterProtocol) -> None:
+        """
+        Replace the requester instance with a new one.
+
+        :param requester: The new requester to be used.
+        """
+        self._requester = requester
+
+    def set_parser(self, parser: ParserProtocol) -> None:
+        """
+        Replace the parser instance with a new one.
+
+        :param parser: The new parser to be used.
+        """
+        self._parser = parser
+
+    def set_saver(self, saver: SaverProtocol) -> None:
+        """
+        Replace the saver instance with a new one.
+
+        :param saver: The new saver to be used.
+        """
+        self._saver = saver

novel_downloader/core/downloaders/common_downloader.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.qidian_downloader
+---------------------------------------------------
+
+This module defines `QidianDownloader`, a platform-specific downloader
+implementation for retrieving novels from Qidian (起点中文网).
+"""
+
+import json
+import logging
+from typing import Any, Dict
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_downloader import BaseDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class CommonDownloader(BaseDownloader):
+    """
+    Specialized downloader for common novels.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        """
+        Initialize the common novel downloader with site information.
+
+        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
+        :param parser: Object implementing ParserProtocol, used to parse page content.
+        :param saver: Object implementing SaverProtocol, used to save final output.
+        :param config: Downloader configuration object.
+        :param site: Identifier for the site the downloader is targeting.
+        """
+        super().__init__(requester, parser, saver, config)
+        self._site = site
+
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        TAG = "[Downloader]"
+        save_html = self.config.save_html
+        skip_existing = self.config.skip_existing
+        site = self.site
+        wait_time = self.config.request_interval
+
+        raw_base = self.raw_data_dir / site / book_id
+        cache_base = self.cache_dir / site / book_id
+        info_path = raw_base / "book_info.json"
+        chapter_dir = raw_base / "chapters"
+        if save_html:
+            chapters_html_dir = cache_base / "html"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+
+        book_info: Dict[str, Any]
+
+        try:
+            if not info_path.exists():
+                raise FileNotFoundError
+            book_info = json.loads(info_path.read_text(encoding="utf-8"))
+            days, hrs, mins, secs = calculate_time_difference(
+                book_info.get("update_time", ""), "UTC+8"
+            )
+            logger.info(
+                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
+            )
+            if days > 1:
+                raise FileNotFoundError  # trigger re-fetch
+        except Exception:
+            info_html = self.requester.get_book_info(book_id, wait_time)
+            if save_html:
+                info_html_path = chapters_html_dir / "info.html"
+                save_as_txt(info_html, info_html_path)
+            book_info = self.parser.parse_book_info(info_html)
+            if (
+                book_info.get("book_name", "") != "未找到书名"
+                and book_info.get("update_time", "") != "未找到更新时间"
+            ):
+                save_as_json(book_info, info_path)
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            cover_bytes = download_image_as_bytes(cover_url, raw_base)
+            if not cover_bytes:
+                logger.warning("%s Failed to download cover: %s", TAG, cover_url)
+
+        # enqueue chapters
+        for vol in book_info.get("volumes", []):
+            vol_name = vol.get("volume_name", "")
+            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
+
+            for chap in vol.get("chapters", []):
+                cid = chap.get("chapterId")
+                if not cid:
+                    logger.warning("%s Skipping chapter without chapterId", TAG)
+                    continue
+
+                chap_path = chapter_dir / f"{cid}.json"
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                chap_title = chap.get("title", "")
+                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
+                try:
+                    chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+
+                    if save_html:
+                        html_path = chapters_html_dir / f"{cid}.html"
+                        save_as_txt(chap_html, html_path, on_exist="skip")
+                        logger.debug(
+                            "%s Saved raw HTML for chapter %s to %s",
+                            TAG,
+                            cid,
+                            html_path,
+                        )
+
+                    chap_json = self.parser.parse_chapter(chap_html, cid)
+                    if not chap_json:
+                        logger.warning(
+                            "%s Parsed chapter json is empty, skipping: %s (%s)",
+                            TAG,
+                            chap_title,
+                            cid,
+                        )
+                        continue
+                except Exception as e:
+                    logger.warning(
+                        "%s Error while processing chapter %s (%s): %s",
+                        TAG,
+                        chap_title,
+                        cid,
+                        str(e),
+                    )
+                    continue
+
+                save_as_json(chap_json, chap_path)
+                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+
+        self.saver.save(book_id)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+    @property
+    def site(self) -> str:
+        """
+        Get the site identifier.
+
+        :return: The site string.
+        """
+        return self._site
+
+    @site.setter
+    def site(self, value: str) -> None:
+        """
+        Set the site identifier.
+
+        :param value: New site string to set.
+        """
+        self._site = value

novel_downloader/core/downloaders/qidian_downloader.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.qidian_downloader
+---------------------------------------------------
+
+This module defines `QidianDownloader`, a platform-specific downloader
+implementation for retrieving novels from Qidian (起点中文网).
+"""
+
+import json
+import logging
+from typing import Any, Dict
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.state import state_mgr
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_downloader import BaseDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class QidianDownloader(BaseDownloader):
+    """
+    Specialized downloader for Qidian novels.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config)
+
+        self._site_key = "qidian"
+        self._is_logged_in = self._handle_login()
+        state_mgr.set_manual_login_flag(self._site_key, not self._is_logged_in)
+
+    def _handle_login(self) -> bool:
+        """
+        Perform login with automatic fallback to manual:
+
+        1. If manual_flag is False, try automatic login:
+           - On success, return True immediately.
+        2. Always attempt manual login if manual_flag is True.
+        3. Return True if manual login succeeds, False otherwise.
+        """
+        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
+
+        # First try automatic login
+        if not manual_flag:
+            if self._requester.login(manual_login=False):
+                return True
+
+        # try manual login
+        return self._requester.login(manual_login=True)
+
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        if not self._is_logged_in:
+            logger.warning(
+                f"[{self._site_key}] login failed, skipping download of {book_id}"
+            )
+            return
+
+        TAG = "[Downloader]"
+        save_html = self.config.save_html
+        skip_existing = self.config.skip_existing
+        wait_time = self.config.request_interval
+
+        raw_base = self.raw_data_dir / "qidian" / book_id
+        cache_base = self.cache_dir / "qidian" / book_id
+        info_path = raw_base / "book_info.json"
+        chapter_dir = raw_base / "chapters"
+        encrypted_chapter_dir = raw_base / "encrypted_chapters"
+        if save_html:
+            chapters_html_dir = cache_base / "html"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+        encrypted_chapter_dir.mkdir(parents=True, exist_ok=True)
+
+        book_info: Dict[str, Any]
+
+        try:
+            if not info_path.exists():
+                raise FileNotFoundError
+            book_info = json.loads(info_path.read_text(encoding="utf-8"))
+            days, hrs, mins, secs = calculate_time_difference(
+                book_info.get("update_time", ""), "UTC+8"
+            )
+            logger.info(
+                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
+            )
+            if days > 1:
+                raise FileNotFoundError  # trigger re-fetch
+        except Exception:
+            info_html = self.requester.get_book_info(book_id, wait_time)
+            if save_html:
+                info_html_path = chapters_html_dir / "info.html"
+                save_as_txt(info_html, info_html_path)
+            book_info = self.parser.parse_book_info(info_html)
+            if (
+                book_info.get("book_name", "") != "未找到书名"
+                and book_info.get("update_time", "") != "未找到更新时间"
+            ):
+                save_as_json(book_info, info_path)
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            cover_bytes = download_image_as_bytes(cover_url, raw_base)
+            if not cover_bytes:
+                logger.warning("%s Failed to download cover: %s", TAG, cover_url)
+
+        # enqueue chapters
+        for vol in book_info.get("volumes", []):
+            vol_name = vol.get("volume_name", "")
+            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
+
+            for chap in vol.get("chapters", []):
+                cid = chap.get("chapterId")
+                if not cid:
+                    logger.warning("%s Skipping chapter without chapterId", TAG)
+                    continue
+
+                chap_path = chapter_dir / f"{cid}.json"
+
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                chap_title = chap.get("title", "")
+                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
+                chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+
+                is_encrypted = self.parser.is_encrypted(chap_html)  # type: ignore[attr-defined]
+
+                folder = encrypted_chapter_dir if is_encrypted else chapter_dir
+                chap_path = folder / f"{cid}.json"
+
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                if save_html and not is_vip(chap_html):
+                    folder = chapters_html_dir / (
+                        "html_encrypted" if is_encrypted else "html_plain"
+                    )
+                    html_path = folder / f"{cid}.html"
+                    save_as_txt(chap_html, html_path, on_exist="skip")
+                    logger.debug(
+                        "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
+                    )
+
+                chap_json = self.parser.parse_chapter(chap_html, cid)
+                if not chap_json:
+                    logger.warning(
+                        "%s Parsed chapter json is empty, skipping: %s (%s)",
+                        TAG,
+                        chap_title,
+                        cid,
+                    )
+                    continue
+
+                save_as_json(chap_json, chap_path)
+                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+
+        self.saver.save(book_id)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+
+def is_vip(html_str: str) -> bool:
+    """
+    Return True if page indicates VIP-only content.
+
+    :param html_str: Raw HTML string.
+    """
+    markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
+    return any(m in html_str for m in markers)

novel_downloader/core/factory/__init__.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.factory
+-----------------------------
+
+This package provides factory methods for dynamically retrieving components
+based on runtime parameters such as site name or content type.
+"""
+
+from .downloader_factory import get_downloader
+from .parser_factory import get_parser
+from .requester_factory import get_requester
+from .saver_factory import get_saver
+
+__all__ = [
+    "get_downloader",
+    "get_parser",
+    "get_requester",
+    "get_saver",
+]

novel_downloader/core/factory/downloader_factory.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.factory.downloader_factory
+------------------------------------------------
+
+This module implements a factory function for creating downloader instances
+based on the site name and parser mode specified in the configuration.
+
+Currently supported:
+- Site: 'qidian'
+    - QidianDownloader
+
+To add support for new sites or modes, extend the `_site_map` accordingly.
+"""
+
+from novel_downloader.config import DownloaderConfig, load_site_rules
+from novel_downloader.core.downloaders import (
+    CommonDownloader,
+    QidianDownloader,
+)
+from novel_downloader.core.interfaces import (
+    DownloaderProtocol,
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+
+_site_map = {
+    "qidian": QidianDownloader,
+    # "biquge": ...
+}
+
+
+def get_downloader(
+    requester: RequesterProtocol,
+    parser: ParserProtocol,
+    saver: SaverProtocol,
+    site: str,
+    config: DownloaderConfig,
+) -> DownloaderProtocol:
+    """
+    Returns a site-specific downloader instance.
+
+    :param requester: Requester implementation
+    :param parser: Parser implementation
+    :param saver: Saver implementation
+    :param site: Site name (e.g., 'qidian')
+    :param config: Downloader configuration
+    :return: An instance of a downloader class
+    """
+    site_key = site.lower()
+
+    if site_key in _site_map:
+        downloader_class = _site_map[site_key]
+        return downloader_class(requester, parser, saver, config)
+
+    site_rules = load_site_rules()
+    if site_key not in site_rules:
+        raise ValueError(f"Unsupported site: {site}")
+
+    return CommonDownloader(requester, parser, saver, config, site_key)