novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/core/downloaders/common_downloader.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.common_downloader
+---------------------------------------------------
+
+This module defines `CommonDownloader`.
+"""
+
+import json
+import logging
+from typing import Any, Dict
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_downloader import BaseDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class CommonDownloader(BaseDownloader):
+    """
+    Specialized downloader for common novels.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+        site: str,
+    ):
+        """
+        Initialize the common novel downloader with site information.
+
+        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
+        :param parser: Object implementing ParserProtocol, used to parse page content.
+        :param saver: Object implementing SaverProtocol, used to save final output.
+        :param config: Downloader configuration object.
+        :param site: Identifier for the site the downloader is targeting.
+        """
+        super().__init__(requester, parser, saver, config)
+        self._site = site
+
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        TAG = "[Downloader]"
+        save_html = self.config.save_html
+        skip_existing = self.config.skip_existing
+        site = self.site
+        wait_time = self.config.request_interval
+
+        raw_base = self.raw_data_dir / site / book_id
+        cache_base = self.cache_dir / site / book_id
+        info_path = raw_base / "book_info.json"
+        chapter_dir = raw_base / "chapters"
+        if save_html:
+            chapters_html_dir = cache_base / "html"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+
+        book_info: Dict[str, Any]
+
+        try:
+            if not info_path.exists():
+                raise FileNotFoundError
+            book_info = json.loads(info_path.read_text(encoding="utf-8"))
+            days, hrs, mins, secs = calculate_time_difference(
+                book_info.get("update_time", ""), "UTC+8"
+            )
+            logger.info(
+                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
+            )
+            if days > 1:
+                raise FileNotFoundError  # trigger re-fetch
+        except Exception:
+            info_html = self.requester.get_book_info(book_id, wait_time)
+            if save_html:
+                info_html_path = chapters_html_dir / "info.html"
+                save_as_txt(info_html, info_html_path)
+            book_info = self.parser.parse_book_info(info_html)
+            if (
+                book_info.get("book_name", "") != "未找到书名"
+                and book_info.get("update_time", "") != "未找到更新时间"
+            ):
+                save_as_json(book_info, info_path)
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            cover_bytes = download_image_as_bytes(cover_url, raw_base)
+            if not cover_bytes:
+                logger.warning("%s Failed to download cover: %s", TAG, cover_url)
+
+        # enqueue chapters
+        for vol in book_info.get("volumes", []):
+            vol_name = vol.get("volume_name", "")
+            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
+
+            for chap in vol.get("chapters", []):
+                cid = chap.get("chapterId")
+                if not cid:
+                    logger.warning("%s Skipping chapter without chapterId", TAG)
+                    continue
+
+                chap_path = chapter_dir / f"{cid}.json"
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                chap_title = chap.get("title", "")
+                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
+                try:
+                    chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+
+                    if save_html:
+                        html_path = chapters_html_dir / f"{cid}.html"
+                        save_as_txt(chap_html, html_path, on_exist="skip")
+                        logger.debug(
+                            "%s Saved raw HTML for chapter %s to %s",
+                            TAG,
+                            cid,
+                            html_path,
+                        )
+
+                    chap_json = self.parser.parse_chapter(chap_html, cid)
+                    if not chap_json:
+                        logger.warning(
+                            "%s Parsed chapter json is empty, skipping: %s (%s)",
+                            TAG,
+                            chap_title,
+                            cid,
+                        )
+                        continue
+                except Exception as e:
+                    logger.warning(
+                        "%s Error while processing chapter %s (%s): %s",
+                        TAG,
+                        chap_title,
+                        cid,
+                        str(e),
+                    )
+                    continue
+
+                save_as_json(chap_json, chap_path)
+                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+
+        self.saver.save(book_id)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+    @property
+    def site(self) -> str:
+        """
+        Get the site identifier.
+
+        :return: The site string.
+        """
+        return self._site
+
+    @site.setter
+    def site(self, value: str) -> None:
+        """
+        Set the site identifier.
+
+        :param value: New site string to set.
+        """
+        self._site = value

novel_downloader/core/downloaders/qidian_downloader.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders.qidian_downloader
+---------------------------------------------------
+
+This module defines `QidianDownloader`, a platform-specific downloader
+implementation for retrieving novels from Qidian (起点中文网).
+"""
+
+import json
+import logging
+from typing import Any, Dict
+
+from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.interfaces import (
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.network import download_image_as_bytes
+from novel_downloader.utils.state import state_mgr
+from novel_downloader.utils.time_utils import calculate_time_difference
+
+from .base_downloader import BaseDownloader
+
+logger = logging.getLogger(__name__)
+
+
+class QidianDownloader(BaseDownloader):
+    """
+    Specialized downloader for Qidian novels.
+    """
+
+    def __init__(
+        self,
+        requester: RequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config)
+
+        self._site_key = "qidian"
+        self._is_logged_in = self._handle_login()
+        state_mgr.set_manual_login_flag(self._site_key, not self._is_logged_in)
+
+    def _handle_login(self) -> bool:
+        """
+        Perform login with automatic fallback to manual:
+
+        1. If manual_flag is False, try automatic login:
+           - On success, return True immediately.
+        2. Always attempt manual login if manual_flag is True.
+        3. Return True if manual login succeeds, False otherwise.
+        """
+        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
+
+        # First try automatic login
+        if not manual_flag:
+            if self._requester.login(manual_login=False):
+                return True
+
+        # try manual login
+        return self._requester.login(manual_login=True)
+
+    def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        if not self._is_logged_in:
+            logger.warning(
+                f"[{self._site_key}] login failed, skipping download of {book_id}"
+            )
+            return
+
+        TAG = "[Downloader]"
+        save_html = self.config.save_html
+        skip_existing = self.config.skip_existing
+        wait_time = self.config.request_interval
+
+        raw_base = self.raw_data_dir / "qidian" / book_id
+        cache_base = self.cache_dir / "qidian" / book_id
+        info_path = raw_base / "book_info.json"
+        chapter_dir = raw_base / "chapters"
+        encrypted_chapter_dir = raw_base / "encrypted_chapters"
+        if save_html:
+            chapters_html_dir = cache_base / "html"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        chapter_dir.mkdir(parents=True, exist_ok=True)
+        encrypted_chapter_dir.mkdir(parents=True, exist_ok=True)
+
+        book_info: Dict[str, Any]
+
+        try:
+            if not info_path.exists():
+                raise FileNotFoundError
+            book_info = json.loads(info_path.read_text(encoding="utf-8"))
+            days, hrs, mins, secs = calculate_time_difference(
+                book_info.get("update_time", ""), "UTC+8"
+            )
+            logger.info(
+                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
+            )
+            if days > 1:
+                raise FileNotFoundError  # trigger re-fetch
+        except Exception:
+            info_html = self.requester.get_book_info(book_id, wait_time)
+            if save_html:
+                info_html_path = chapters_html_dir / "info.html"
+                save_as_txt(info_html, info_html_path)
+            book_info = self.parser.parse_book_info(info_html)
+            if (
+                book_info.get("book_name", "") != "未找到书名"
+                and book_info.get("update_time", "") != "未找到更新时间"
+            ):
+                save_as_json(book_info, info_path)
+
+        # download cover
+        cover_url = book_info.get("cover_url", "")
+        if cover_url:
+            cover_bytes = download_image_as_bytes(cover_url, raw_base)
+            if not cover_bytes:
+                logger.warning("%s Failed to download cover: %s", TAG, cover_url)
+
+        # enqueue chapters
+        for vol in book_info.get("volumes", []):
+            vol_name = vol.get("volume_name", "")
+            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
+
+            for chap in vol.get("chapters", []):
+                cid = chap.get("chapterId")
+                if not cid:
+                    logger.warning("%s Skipping chapter without chapterId", TAG)
+                    continue
+
+                chap_path = chapter_dir / f"{cid}.json"
+
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                chap_title = chap.get("title", "")
+                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
+                chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+
+                is_encrypted = self.parser.is_encrypted(chap_html)  # type: ignore[attr-defined]
+
+                folder = encrypted_chapter_dir if is_encrypted else chapter_dir
+                chap_path = folder / f"{cid}.json"
+
+                if chap_path.exists() and skip_existing:
+                    logger.debug(
+                        "%s Chapter already exists, skipping: %s",
+                        TAG,
+                        cid,
+                    )
+                    continue
+
+                if save_html and not is_vip(chap_html):
+                    folder = chapters_html_dir / (
+                        "html_encrypted" if is_encrypted else "html_plain"
+                    )
+                    html_path = folder / f"{cid}.html"
+                    save_as_txt(chap_html, html_path, on_exist="skip")
+                    logger.debug(
+                        "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
+                    )
+
+                chap_json = self.parser.parse_chapter(chap_html, cid)
+                if not chap_json:
+                    logger.warning(
+                        "%s Parsed chapter json is empty, skipping: %s (%s)",
+                        TAG,
+                        chap_title,
+                        cid,
+                    )
+                    continue
+
+                save_as_json(chap_json, chap_path)
+                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+
+        self.saver.save(book_id)
+
+        logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+
+def is_vip(html_str: str) -> bool:
+    """
+    Return True if page indicates VIP-only content.
+
+    :param html_str: Raw HTML string.
+    """
+    markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
+    return any(m in html_str for m in markers)

novel_downloader/core/factory/__init__.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.factory
+-----------------------------
+
+This package provides factory methods for dynamically retrieving components
+based on runtime parameters such as site name or content type.
+"""
+
+from .downloader_factory import (
+    get_async_downloader,
+    get_downloader,
+    get_sync_downloader,
+)
+from .parser_factory import get_parser
+from .requester_factory import (
+    get_async_requester,
+    get_requester,
+    get_sync_requester,
+)
+from .saver_factory import get_saver
+
+__all__ = [
+    "get_async_downloader",
+    "get_downloader",
+    "get_sync_downloader",
+    "get_parser",
+    "get_async_requester",
+    "get_requester",
+    "get_sync_requester",
+    "get_saver",
+]

novel_downloader/core/factory/downloader_factory.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.factory.downloader_factory
+------------------------------------------------
+
+This module implements a factory function for creating downloader instances
+based on the site name and parser mode specified in the configuration.
+
+- get_async_downloader -> always returns a CommonAsyncDownloader
+- get_sync_downloader  -> returns a site-specific downloader or CommonDownloader
+- get_downloader       -> dispatches to one of the above based on config.mode
+
+To add support for new sites or modes, extend the `_site_map` accordingly.
+"""
+
+from typing import Union
+
+from novel_downloader.config import DownloaderConfig, load_site_rules
+from novel_downloader.core.downloaders import (
+    CommonAsyncDownloader,
+    CommonDownloader,
+    QidianDownloader,
+)
+from novel_downloader.core.interfaces import (
+    AsyncDownloaderProtocol,
+    AsyncRequesterProtocol,
+    DownloaderProtocol,
+    ParserProtocol,
+    RequesterProtocol,
+    SaverProtocol,
+)
+
+_site_map = {
+    "qidian": QidianDownloader,
+    # "biquge": ...
+}
+
+
+def get_async_downloader(
+    requester: AsyncRequesterProtocol,
+    parser: ParserProtocol,
+    saver: SaverProtocol,
+    site: str,
+    config: DownloaderConfig,
+) -> AsyncDownloaderProtocol:
+    """
+    Returns an AsyncDownloaderProtocol for the given site.
+
+    :param requester: Requester implementation
+    :param parser: Parser implementation
+    :param saver: Saver implementation
+    :param site: Site name (e.g., 'qidian')
+    :param config: Downloader configuration
+
+    :return: An instance of a downloader class
+
+    :raises ValueError: If a site-specific downloader does not support async mode.
+    :raises TypeError: If the provided requester does not match the required protocol
+                    for the chosen mode (sync vs async).
+    """
+    site_key = site.lower()
+
+    if not isinstance(requester, AsyncRequesterProtocol):
+        raise TypeError("Async mode requires an AsyncRequesterProtocol")
+
+    site_rules = load_site_rules()
+    site_rule = site_rules.get(site_key)
+    if site_rule is None:
+        raise ValueError(f"Unsupported site: {site}")
+
+    return CommonAsyncDownloader(requester, parser, saver, config, site_key)
+
+
+def get_sync_downloader(
+    requester: RequesterProtocol,
+    parser: ParserProtocol,
+    saver: SaverProtocol,
+    site: str,
+    config: DownloaderConfig,
+) -> DownloaderProtocol:
+    """
+    Returns a DownloaderProtocol for the given site.
+    First tries a site-specific downloader (e.g. QidianDownloader),
+    otherwise falls back to CommonDownloader.
+
+    :param requester: Requester implementation
+    :param parser: Parser implementation
+    :param saver: Saver implementation
+    :param site: Site name (e.g., 'qidian')
+    :param config: Downloader configuration
+
+    :return: An instance of a downloader class
+
+    :raises ValueError: If a site-specific downloader does not support async mode.
+    :raises TypeError: If the provided requester does not match the required protocol
+                    for the chosen mode (sync vs async).
+    """
+    site_key = site.lower()
+
+    if not isinstance(requester, RequesterProtocol):
+        raise TypeError("Sync mode requires a RequesterProtocol")
+
+    # site-specific
+    if site_key in _site_map:
+        return _site_map[site_key](requester, parser, saver, config)
+
+    # fallback
+    site_rules = load_site_rules()
+    site_rule = site_rules.get(site_key)
+    if site_rule is None:
+        raise ValueError(f"Unsupported site: {site}")
+
+    return CommonDownloader(requester, parser, saver, config, site_key)
+
+
+def get_downloader(
+    requester: Union[AsyncRequesterProtocol, RequesterProtocol],
+    parser: ParserProtocol,
+    saver: SaverProtocol,
+    site: str,
+    config: DownloaderConfig,
+) -> Union[AsyncDownloaderProtocol, DownloaderProtocol]:
+    """
+    Dispatches to get_async_downloader if config.mode == 'async',
+    otherwise to get_sync_downloader.
+
+    :param requester: Requester implementation
+    :param parser: Parser implementation
+    :param saver: Saver implementation
+    :param site: Site name (e.g., 'qidian')
+    :param config: Downloader configuration
+
+    :return: An instance of a downloader class
+
+    :raises ValueError: If a site-specific downloader does not support async mode.
+    :raises TypeError: If the provided requester does not match the required protocol
+                    for the chosen mode (sync vs async).
+    """
+    mode = config.mode.lower()
+    if mode == "async":
+        if not isinstance(requester, AsyncRequesterProtocol):
+            raise TypeError("Async mode requires an AsyncRequesterProtocol")
+        return get_async_downloader(requester, parser, saver, site, config)
+    if mode in ("browser", "session"):
+        if not isinstance(requester, RequesterProtocol):
+            raise TypeError("Sync mode requires a RequesterProtocol")
+        return get_sync_downloader(requester, parser, saver, site, config)
+    raise ValueError(f"Unknown mode '{config.mode}' for site '{site}'")

novel_downloader/core/factory/parser_factory.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.factory.parser_factory
+--------------------------------------------
+
+This module implements a factory function for creating parser instances
+based on the site name and parser mode specified in the configuration.
+
+Currently supported:
+- Site: 'qidian'
+    - Modes:
+        - 'browser': QidianBrowserParser
+        - 'session': (Not implemented yet)
+
+To add support for new sites or modes, extend the `_site_map` accordingly.
+"""
+
+from novel_downloader.config import ParserConfig, load_site_rules
+from novel_downloader.core.interfaces import ParserProtocol
+from novel_downloader.core.parsers import (
+    CommonParser,
+    QidianBrowserParser,
+    QidianSessionParser,
+)
+
+_site_map = {
+    "qidian": {
+        "browser": QidianBrowserParser,
+        "session": QidianSessionParser,
+    },
+    # "biquge": ...
+}
+
+
+def get_parser(site: str, config: ParserConfig) -> ParserProtocol:
+    """
+    Returns a site-specific parser instance.
+
+    :param site: Site name (e.g., 'qidian')
+    :param config: Configuration for the parser
+    :return: An instance of a parser class
+    """
+    site_key = site.lower()
+
+    if site_key in _site_map:
+        site_entry = _site_map[site_key]
+        if isinstance(site_entry, dict):
+            parser_class = site_entry.get(config.mode)
+            if parser_class is None:
+                raise ValueError(f"Unsupported mode '{config.mode}' for site '{site}'")
+        else:
+            parser_class = site_entry
+        return parser_class(config)
+
+    # Fallback: site not mapped specially, try to load rule
+    site_rules = load_site_rules()
+    site_rule = site_rules.get(site_key)
+    if site_rule is None:
+        raise ValueError(f"Unsupported site: {site}")
+
+    return CommonParser(config, site_key, site_rule)