novel-downloader 1.3.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/core/downloaders/qidian/qidian_sync.py

@@ -1,227 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qidian.qidian_sync
-----------------------------------------------------
-
-This module defines `QidianDownloader`, a platform-specific downloader
-implementation for retrieving novels from Qidian (起点中文网).
-"""
-
-import json
-from typing import Any
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.downloaders.base import BaseDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-from novel_downloader.utils.chapter_storage import ChapterStorage
-from novel_downloader.utils.file_utils import save_as_json, save_as_txt
-from novel_downloader.utils.network import download_image_as_bytes
-from novel_downloader.utils.state import state_mgr
-from novel_downloader.utils.time_utils import (
-    calculate_time_difference,
-    sleep_with_random_delay,
-)
-
-
-class QidianDownloader(BaseDownloader):
-    """
-    Specialized downloader for Qidian novels.
-    """
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "qidian")
-
-        self._site_key = "qidian"
-        self._is_logged_in = self._handle_login()
-        state_mgr.set_manual_login_flag(self._site_key, not self._is_logged_in)
-
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        :param book_id: The identifier of the book to download.
-        """
-        if not self._is_logged_in:
-            self.logger.warning(
-                "[%s] login failed, skipping download of %s",
-                self._site_key,
-                book_id,
-            )
-            return
-
-        TAG = "[Downloader]"
-        save_html = self.config.save_html
-        skip_existing = self.config.skip_existing
-        wait_time = self.config.request_interval
-        scroll = self.config.mode == "browser"
-
-        raw_base = self.raw_data_dir / book_id
-        cache_base = self.cache_dir / book_id
-        info_path = raw_base / "book_info.json"
-        chapters_html_dir = cache_base / "html"
-
-        raw_base.mkdir(parents=True, exist_ok=True)
-        normal_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-        encrypted_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="encrypted_chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-
-        book_info: dict[str, Any]
-
-        try:
-            if not info_path.exists():
-                raise FileNotFoundError
-            book_info = json.loads(info_path.read_text(encoding="utf-8"))
-            days, hrs, mins, secs = calculate_time_difference(
-                book_info.get("update_time", ""), "UTC+8"
-            )
-            self.logger.info(
-                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
-            )
-            if days > 1:
-                raise FileNotFoundError  # trigger re-fetch
-        except Exception:
-            info_html = self.requester.get_book_info(book_id)
-            if save_html and info_html:
-                info_html_path = chapters_html_dir / "info.html"
-                save_as_txt(info_html[0], info_html_path)
-            book_info = self.parser.parse_book_info(info_html)
-            if (
-                book_info.get("book_name", "") != "未找到书名"
-                and book_info.get("update_time", "") != "未找到更新时间"
-            ):
-                save_as_json(book_info, info_path)
-            sleep_with_random_delay(wait_time, mul_spread=1.1, max_sleep=wait_time + 2)
-
-        # download cover
-        cover_url = book_info.get("cover_url", "")
-        if cover_url:
-            cover_bytes = download_image_as_bytes(cover_url, raw_base)
-            if not cover_bytes:
-                self.logger.warning("%s Failed to download cover: %s", TAG, cover_url)
-
-        # enqueue chapters
-        for vol in book_info.get("volumes", []):
-            vol_name = vol.get("volume_name", "")
-            self.logger.info("%s Enqueuing volume: %s", TAG, vol_name)
-
-            for chap in vol.get("chapters", []):
-                cid = chap.get("chapterId")
-                if not cid:
-                    self.logger.warning("%s Skipping chapter without chapterId", TAG)
-                    continue
-
-                if normal_cs.exists(cid) and skip_existing:
-                    self.logger.debug(
-                        "%s Chapter already exists, skipping: %s",
-                        TAG,
-                        cid,
-                    )
-                    continue
-
-                chap_title = chap.get("title", "")
-                self.logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
-                chap_html = self.requester.get_book_chapter(book_id, cid)
-                if not chap_html:
-                    continue
-
-                if scroll:
-                    self.requester.scroll_page(wait_time * 2)  # type: ignore[attr-defined]
-                else:
-                    sleep_with_random_delay(
-                        wait_time, mul_spread=1.1, max_sleep=wait_time + 2
-                    )
-
-                is_encrypted = self.parser.is_encrypted(chap_html[0])  # type: ignore[attr-defined]
-
-                if is_encrypted and encrypted_cs.exists(cid) and skip_existing:
-                    self.logger.debug(
-                        "%s Chapter already exists, skipping: %s",
-                        TAG,
-                        cid,
-                    )
-                    continue
-
-                if save_html and chap_html and not is_vip(chap_html[0]):
-                    folder = chapters_html_dir / (
-                        "html_encrypted" if is_encrypted else "html_plain"
-                    )
-                    html_path = folder / f"{cid}.html"
-                    save_as_txt(chap_html[0], html_path, on_exist="skip")
-                    self.logger.debug(
-                        "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
-                    )
-
-                chap_json = self.parser.parse_chapter(chap_html, cid)
-                if not chap_json or not chap_json.get("content"):
-                    self.logger.warning(
-                        "%s Parsed chapter json is empty, skipping: %s (%s)",
-                        TAG,
-                        chap_title,
-                        cid,
-                    )
-                    continue
-
-                if is_encrypted:
-                    encrypted_cs.save(chap_json)
-                else:
-                    normal_cs.save(chap_json)
-                self.logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
-
-        normal_cs.close()
-        encrypted_cs.close()
-
-        self.saver.save(book_id)
-
-        self.logger.info(
-            "%s Novel '%s' download completed.",
-            TAG,
-            book_info.get("book_name", "unknown"),
-        )
-        return
-
-    def _handle_login(self) -> bool:
-        """
-        Perform login with automatic fallback to manual:
-
-        1. If manual_flag is False, try automatic login:
-           - On success, return True immediately.
-        2. Always attempt manual login if manual_flag is True.
-        3. Return True if manual login succeeds, False otherwise.
-        """
-        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
-
-        # First try automatic login
-        if not manual_flag and self._requester.login(manual_login=False):
-            return True
-
-        # try manual login
-        return self._requester.login(manual_login=True)
-
-
-def is_vip(html_str: str) -> bool:
-    """
-    Return True if page indicates VIP-only content.
-
-    :param html_str: Raw HTML string.
-    """
-    markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
-    return any(m in html_str for m in markers)

novel_downloader/core/downloaders/sfacg/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg
----------------------------------------
-
-"""
-
-from .sfacg_async import SfacgAsyncDownloader
-from .sfacg_sync import SfacgDownloader
-
-__all__ = [
-    "SfacgAsyncDownloader",
-    "SfacgDownloader",
-]

novel_downloader/core/downloaders/sfacg/sfacg_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg.sfacg_async
----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class SfacgAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/sfacg/sfacg_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg.sfacg_sync
---------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class SfacgDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/yamibo/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.yamibo
-----------------------------------------
-
-"""
-
-from .yamibo_async import YamiboAsyncDownloader
-from .yamibo_sync import YamiboDownloader
-
-__all__ = [
-    "YamiboAsyncDownloader",
-    "YamiboDownloader",
-]

novel_downloader/core/downloaders/yamibo/yamibo_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.yamibo.yamibo_async
------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class YamiboAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "yamibo")

novel_downloader/core/downloaders/yamibo/yamibo_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.yamibo.yamibo_sync
-----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class YamiboDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "yamibo")

novel_downloader/core/factory/requester.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.factory.requester_factory
------------------------------------------------
-
-This module implements a factory function for retrieving requester instances
-based on the target novel platform (site).
-"""
-
-from collections.abc import Callable
-
-from novel_downloader.config import RequesterConfig, load_site_rules
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    SyncRequesterProtocol,
-)
-from novel_downloader.core.requesters import (
-    BiqugeAsyncSession,
-    BiqugeSession,
-    CommonAsyncSession,
-    CommonSession,
-    EsjzoneAsyncSession,
-    EsjzoneSession,
-    QianbiAsyncSession,
-    QianbiSession,
-    QidianBrowser,
-    QidianSession,
-    SfacgAsyncSession,
-    SfacgSession,
-    YamiboAsyncSession,
-    YamiboSession,
-)
-
-AsyncRequesterBuilder = Callable[[RequesterConfig], AsyncRequesterProtocol]
-SyncRequesterBuilder = Callable[[RequesterConfig], SyncRequesterProtocol]
-
-
-_async_site_map: dict[str, AsyncRequesterBuilder] = {
-    "biquge": BiqugeAsyncSession,
-    "esjzone": EsjzoneAsyncSession,
-    "qianbi": QianbiAsyncSession,
-    "sfacg": SfacgAsyncSession,
-    "yamibo": YamiboAsyncSession,
-}
-_sync_site_map: dict[
-    str,
-    dict[str, SyncRequesterBuilder],
-] = {
-    "biquge": {
-        "session": BiqugeSession,
-    },
-    "esjzone": {
-        "session": EsjzoneSession,
-    },
-    "qianbi": {
-        "session": QianbiSession,
-    },
-    "qidian": {
-        "session": QidianSession,
-        "browser": QidianBrowser,
-    },
-    "sfacg": {
-        "session": SfacgSession,
-    },
-    "yamibo": {
-        "session": YamiboSession,
-    },
-}
-
-
-def get_async_requester(
-    site: str,
-    config: RequesterConfig,
-) -> AsyncRequesterProtocol:
-    """
-    Returns an AsyncRequesterProtocol for the given site.
-
-    :param site: Site name (e.g., 'qidian')
-    :param config: Configuration for the requester
-    :return: An instance of a requester class
-    """
-    site_key = site.lower()
-
-    # site-specific
-    if site_key in _async_site_map:
-        return _async_site_map[site_key](config)
-
-    # fallback
-    site_rules = load_site_rules()
-    site_rule = site_rules.get(site_key)
-    if site_rule is None:
-        raise ValueError(f"Unsupported site: {site}")
-    profile = site_rule["profile"]
-    return CommonAsyncSession(config, site_key, profile)
-
-
-def get_sync_requester(
-    site: str,
-    config: RequesterConfig,
-) -> SyncRequesterProtocol:
-    """
-    Returns a RequesterProtocol for the given site.
-
-    :param site: Site name (e.g., 'qidian')
-    :param config: Configuration for the requester
-    :return: An instance of a requester class
-    """
-    site_key = site.lower()
-    site_entry = _sync_site_map.get(site_key)
-
-    # site-specific
-    if site_entry:
-        cls = site_entry.get(config.mode)
-        if cls:
-            return cls(config)
-
-    # fallback
-    site_rules = load_site_rules()
-    site_rule = site_rules.get(site_key)
-    if site_rule is None:
-        raise ValueError(f"Unsupported site: {site}")
-    profile = site_rule["profile"]
-    return CommonSession(config, site_key, profile)
-
-
-def get_requester(
-    site: str,
-    config: RequesterConfig,
-) -> AsyncRequesterProtocol | SyncRequesterProtocol:
-    """
-    Dispatches to either get_async_requester or get_sync_requester
-    based on config.mode. Treats 'browser' and 'async' as async modes,
-    'session' as sync; anything else is an error.
-
-    :param site: Site name (e.g., 'qidian')
-    :param config: Configuration for the requester
-    :return: An instance of a requester class
-    """
-    mode = config.mode.lower()
-    if mode == "async":
-        return get_async_requester(site, config)
-    if mode in ("browser", "session"):
-        return get_sync_requester(site, config)
-    raise ValueError(f"Unknown mode '{config.mode}' for site '{site}'")

novel_downloader/core/factory/saver.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.factory.parser_factory
---------------------------------------------
-
-This module implements a factory function for creating saver instances
-based on the site name and parser mode specified in the configuration.
-"""
-
-from collections.abc import Callable
-
-from novel_downloader.config import SaverConfig, load_site_rules
-from novel_downloader.core.interfaces import SaverProtocol
-from novel_downloader.core.savers import (
-    BiqugeSaver,
-    CommonSaver,
-    EsjzoneSaver,
-    QianbiSaver,
-    QidianSaver,
-    SfacgSaver,
-    YamiboSaver,
-)
-
-SaverBuilder = Callable[[SaverConfig], SaverProtocol]
-
-_site_map: dict[str, SaverBuilder] = {
-    "biquge": BiqugeSaver,
-    "esjzone": EsjzoneSaver,
-    "qianbi": QianbiSaver,
-    "qidian": QidianSaver,
-    "sfacg": SfacgSaver,
-    "yamibo": YamiboSaver,
-}
-
-
-def get_saver(site: str, config: SaverConfig) -> SaverProtocol:
-    """
-    Returns a site-specific saver instance.
-
-    :param site: Site name (e.g., 'qidian')
-    :param config: Configuration for the saver
-    :return: An instance of a saver class
-    """
-    site_key = site.lower()
-
-    # site-specific
-    saver_class = _site_map.get(site_key)
-    if saver_class:
-        return saver_class(config)
-
-    # Fallback
-    site_rules = load_site_rules()
-    if site_key not in site_rules:
-        raise ValueError(f"Unsupported site: {site}")
-
-    return CommonSaver(config, site_key)

novel_downloader/core/interfaces/async_downloader.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.interfaces.async_downloader
-----------------------------------------------------------
-
-This module defines the AsyncDownloaderProtocol, a structural interface
-that outlines the expected behavior of any downloader class.
-"""
-
-from typing import Protocol
-
-
-class AsyncDownloaderProtocol(Protocol):
-    """
-    Protocol for fully-asynchronous downloader classes.
-
-    Defines the expected interface for any downloader implementation,
-    including both batch and single book downloads,
-    as well as optional pre-download hooks.
-    """
-
-    async def download(self, book_ids: list[str]) -> None:
-        """
-        Batch download entry point.
-
-        :param book_ids: List of book IDs to download.
-        """
-        ...
-
-    async def download_one(self, book_id: str) -> None:
-        """
-        Download logic for a single book.
-
-        :param book_id: The identifier of the book.
-        """
-        ...

novel_downloader/core/interfaces/async_requester.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.interfaces.async_requester
---------------------------------------------------------
-
-Defines the AsyncRequesterProtocol interface for fetching raw HTML or JSON
-for book info pages, individual chapters, managing request lifecycle,
-and optionally retrieving a user's authenticated bookcase.
-"""
-
-from typing import Any, Literal, Protocol, runtime_checkable
-
-
-@runtime_checkable
-class AsyncRequesterProtocol(Protocol):
-    """
-    An async requester must be able to fetch raw HTML/data for:
-      - a book's info page,
-      - a specific chapter page,
-    and manage login/shutdown asynchronously.
-    """
-
-    def is_async(self) -> Literal[True]:
-        ...
-
-    async def login(
-        self,
-        username: str = "",
-        password: str = "",
-        manual_login: bool = False,
-        **kwargs: Any,
-    ) -> bool:
-        """
-        Attempt to log in asynchronously.
-        :returns: True if login succeeded.
-        """
-        ...
-
-    async def get_book_info(
-        self,
-        book_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML (or JSON) of the book info page asynchronously.
-
-        :param book_id: The book identifier.
-        :return: The page content as a string.
-        """
-        ...
-
-    async def get_book_chapter(
-        self,
-        book_id: str,
-        chapter_id: str,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Fetch the raw HTML (or JSON) of a single chapter asynchronously.
-
-        :param book_id: The book identifier.
-        :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
-        """
-        ...
-
-    async def get_bookcase(
-        self,
-        page: int = 1,
-        **kwargs: Any,
-    ) -> list[str]:
-        """
-        Optional: Retrieve the HTML content of the authenticated
-        user's bookcase page asynchronously.
-
-        :return: The HTML markup of the bookcase page.
-        """
-        ...
-
-    async def close(self) -> None:
-        """
-        Shutdown and clean up any resources (e.g., close aiohttp session).
-        """
-        ...