novel-downloader 1.3.3__py3-none-any.whl → 1.4.1__py3-none-any.whl

novel_downloader/core/downloaders/common/common_sync.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.common.common_sync
-----------------------------------------------------
-
-This module defines `CommonDownloader`.
-"""
-
-import json
-import logging
-from typing import Any
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.downloaders.base import BaseDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-from novel_downloader.utils.chapter_storage import ChapterStorage
-from novel_downloader.utils.file_utils import save_as_json, save_as_txt
-from novel_downloader.utils.time_utils import (
-    calculate_time_difference,
-    sleep_with_random_delay,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class CommonDownloader(BaseDownloader):
-    """
-    Specialized downloader for common novels.
-    """
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        """
-        Initialize the common novel downloader with site information.
-
-        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
-        :param parser: Object implementing ParserProtocol, used to parse page content.
-        :param saver: Object implementing SaverProtocol, used to save final output.
-        :param config: Downloader configuration object.
-        :param site: Identifier for the site the downloader is targeting.
-        """
-        super().__init__(requester, parser, saver, config, site)
-        self._site = site
-        self._is_logged_in = False
-
-    def prepare(self) -> None:
-        """
-        Perform login
-        """
-        if self.login_required and not self._is_logged_in:
-            success = self.requester.login()
-            if not success:
-                raise RuntimeError("Login failed")
-            self._is_logged_in = True
-
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        :param book_id: The identifier of the book to download.
-        """
-        self.prepare()
-
-        TAG = "[Downloader]"
-        save_html = self.config.save_html
-        skip_existing = self.config.skip_existing
-        wait_time = self.config.request_interval
-
-        raw_base = self.raw_data_dir / book_id
-        cache_base = self.cache_dir / book_id
-        info_path = raw_base / "book_info.json"
-        chapters_html_dir = cache_base / "html"
-
-        raw_base.mkdir(parents=True, exist_ok=True)
-        if self.save_html:
-            chapters_html_dir.mkdir(parents=True, exist_ok=True)
-        normal_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-
-        book_info: dict[str, Any]
-
-        try:
-            if not info_path.exists():
-                raise FileNotFoundError
-            book_info = json.loads(info_path.read_text(encoding="utf-8"))
-            days, hrs, mins, secs = calculate_time_difference(
-                book_info.get("update_time", ""), "UTC+8"
-            )
-            logger.info(
-                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
-            )
-            if days > 1:
-                raise FileNotFoundError  # trigger re-fetch
-        except Exception:
-            info_html = self.requester.get_book_info(book_id)
-            if save_html:
-                for i, html in enumerate(info_html):
-                    save_as_txt(html, chapters_html_dir / f"info_{i}.html")
-            book_info = self.parser.parse_book_info(info_html)
-            if (
-                book_info.get("book_name", "") != "未找到书名"
-                and book_info.get("update_time", "") != "未找到更新时间"
-            ):
-                save_as_json(book_info, info_path)
-            sleep_with_random_delay(wait_time, mul_spread=1.1, max_sleep=wait_time + 2)
-
-        # enqueue chapters
-        for vol in book_info.get("volumes", []):
-            vol_name = vol.get("volume_name", "")
-            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
-
-            for chap in vol.get("chapters", []):
-                cid = chap.get("chapterId")
-                if not cid:
-                    logger.warning("%s Skipping chapter without chapterId", TAG)
-                    continue
-
-                if normal_cs.exists(cid) and skip_existing:
-                    logger.debug(
-                        "%s Chapter already exists, skipping: %s",
-                        TAG,
-                        cid,
-                    )
-                    continue
-
-                chap_title = chap.get("title", "")
-                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
-                try:
-                    chap_html = self.requester.get_book_chapter(book_id, cid)
-
-                    if save_html:
-                        for i, html in enumerate(chap_html):
-                            html_path = chapters_html_dir / f"{cid}_{i}.html"
-                            save_as_txt(html, html_path, on_exist="skip")
-
-                    chap_json = self.parser.parse_chapter(chap_html, cid)
-
-                    sleep_with_random_delay(
-                        wait_time, mul_spread=1.1, max_sleep=wait_time + 2
-                    )
-                    if not chap_json:
-                        logger.warning(
-                            "%s Parsed chapter json is empty, skipping: %s (%s)",
-                            TAG,
-                            chap_title,
-                            cid,
-                        )
-                        continue
-                except Exception as e:
-                    logger.warning(
-                        "%s Error while processing chapter %s (%s): %s",
-                        TAG,
-                        chap_title,
-                        cid,
-                        str(e),
-                    )
-                    continue
-
-                normal_cs.save(chap_json)
-                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
-
-        normal_cs.close()
-        self.saver.save(book_id)
-
-        logger.info(
-            "%s Novel '%s' download completed.",
-            TAG,
-            book_info.get("book_name", "unknown"),
-        )
-        return
-
-    @property
-    def site(self) -> str:
-        """
-        Get the site identifier.
-
-        :return: The site string.
-        """
-        return self._site
-
-    @site.setter
-    def site(self, value: str) -> None:
-        """
-        Set the site identifier.
-
-        :param value: New site string to set.
-        """
-        self._site = value

novel_downloader/core/downloaders/esjzone/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.esjzone
------------------------------------------
-
-"""
-
-from .esjzone_async import EsjzoneAsyncDownloader
-from .esjzone_sync import EsjzoneDownloader
-
-__all__ = [
-    "EsjzoneAsyncDownloader",
-    "EsjzoneDownloader",
-]

novel_downloader/core/downloaders/esjzone/esjzone_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.esjzone.esjzone_async
--------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class EsjzoneAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "esjzone")

novel_downloader/core/downloaders/esjzone/esjzone_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.esjzone.esjzone_sync
-------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class EsjzoneDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "esjzone")

novel_downloader/core/downloaders/qianbi/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qianbi
-----------------------------------------
-
-"""
-
-from .qianbi_async import QianbiAsyncDownloader
-from .qianbi_sync import QianbiDownloader
-
-__all__ = [
-    "QianbiAsyncDownloader",
-    "QianbiDownloader",
-]

novel_downloader/core/downloaders/qianbi/qianbi_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qianbi.qianbi_async
------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class QianbiAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "qianbi")

novel_downloader/core/downloaders/qianbi/qianbi_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qianbi.qianbi_sync
-----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class QianbiDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "qianbi")

novel_downloader/core/downloaders/qidian/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qidian
-----------------------------------------
-
-"""
-
-from .qidian_sync import QidianDownloader
-
-__all__ = ["QidianDownloader"]

novel_downloader/core/downloaders/qidian/qidian_sync.py

@@ -1,219 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.qidian.qidian_sync
-----------------------------------------------------
-
-This module defines `QidianDownloader`, a platform-specific downloader
-implementation for retrieving novels from Qidian (起点中文网).
-"""
-
-import json
-from typing import Any
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.downloaders.base import BaseDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-from novel_downloader.utils.chapter_storage import ChapterStorage
-from novel_downloader.utils.file_utils import save_as_json, save_as_txt
-from novel_downloader.utils.state import state_mgr
-from novel_downloader.utils.time_utils import (
-    calculate_time_difference,
-    sleep_with_random_delay,
-)
-
-
-class QidianDownloader(BaseDownloader):
-    """
-    Specialized downloader for Qidian novels.
-    """
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "qidian")
-
-        self._site_key = "qidian"
-        self._is_logged_in = self._handle_login()
-        state_mgr.set_manual_login_flag(self._site_key, not self._is_logged_in)
-
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        :param book_id: The identifier of the book to download.
-        """
-        if not self._is_logged_in:
-            self.logger.warning(
-                "[%s] login failed, skipping download of %s",
-                self._site_key,
-                book_id,
-            )
-            return
-
-        TAG = "[Downloader]"
-        save_html = self.config.save_html
-        skip_existing = self.config.skip_existing
-        wait_time = self.config.request_interval
-        scroll = self.config.mode == "browser"
-
-        raw_base = self.raw_data_dir / book_id
-        cache_base = self.cache_dir / book_id
-        info_path = raw_base / "book_info.json"
-        chapters_html_dir = cache_base / "html"
-
-        raw_base.mkdir(parents=True, exist_ok=True)
-        normal_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-        encrypted_cs = ChapterStorage(
-            raw_base=raw_base,
-            namespace="encrypted_chapters",
-            backend_type=self._config.storage_backend,
-            batch_size=self._config.storage_batch_size,
-        )
-
-        book_info: dict[str, Any]
-
-        try:
-            if not info_path.exists():
-                raise FileNotFoundError
-            book_info = json.loads(info_path.read_text(encoding="utf-8"))
-            days, hrs, mins, secs = calculate_time_difference(
-                book_info.get("update_time", ""), "UTC+8"
-            )
-            self.logger.info(
-                "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
-            )
-            if days > 1:
-                raise FileNotFoundError  # trigger re-fetch
-        except Exception:
-            info_html = self.requester.get_book_info(book_id)
-            if save_html and info_html:
-                info_html_path = chapters_html_dir / "info.html"
-                save_as_txt(info_html[0], info_html_path)
-            book_info = self.parser.parse_book_info(info_html)
-            if (
-                book_info.get("book_name", "") != "未找到书名"
-                and book_info.get("update_time", "") != "未找到更新时间"
-            ):
-                save_as_json(book_info, info_path)
-            sleep_with_random_delay(wait_time, mul_spread=1.1, max_sleep=wait_time + 2)
-
-        # enqueue chapters
-        for vol in book_info.get("volumes", []):
-            vol_name = vol.get("volume_name", "")
-            self.logger.info("%s Enqueuing volume: %s", TAG, vol_name)
-
-            for chap in vol.get("chapters", []):
-                cid = chap.get("chapterId")
-                if not cid:
-                    self.logger.warning("%s Skipping chapter without chapterId", TAG)
-                    continue
-
-                if normal_cs.exists(cid) and skip_existing:
-                    self.logger.debug(
-                        "%s Chapter already exists, skipping: %s",
-                        TAG,
-                        cid,
-                    )
-                    continue
-
-                chap_title = chap.get("title", "")
-                self.logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
-                chap_html = self.requester.get_book_chapter(book_id, cid)
-                if not chap_html:
-                    continue
-
-                if scroll:
-                    self.requester.scroll_page(wait_time * 2)  # type: ignore[attr-defined]
-                else:
-                    sleep_with_random_delay(
-                        wait_time, mul_spread=1.1, max_sleep=wait_time + 2
-                    )
-
-                is_encrypted = self.parser.is_encrypted(chap_html[0])  # type: ignore[attr-defined]
-
-                if is_encrypted and encrypted_cs.exists(cid) and skip_existing:
-                    self.logger.debug(
-                        "%s Chapter already exists, skipping: %s",
-                        TAG,
-                        cid,
-                    )
-                    continue
-
-                if save_html and chap_html and not is_vip(chap_html[0]):
-                    folder = chapters_html_dir / (
-                        "html_encrypted" if is_encrypted else "html_plain"
-                    )
-                    html_path = folder / f"{cid}.html"
-                    save_as_txt(chap_html[0], html_path, on_exist="skip")
-                    self.logger.debug(
-                        "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
-                    )
-
-                chap_json = self.parser.parse_chapter(chap_html, cid)
-                if not chap_json or not chap_json.get("content"):
-                    self.logger.warning(
-                        "%s Parsed chapter json is empty, skipping: %s (%s)",
-                        TAG,
-                        chap_title,
-                        cid,
-                    )
-                    continue
-
-                if is_encrypted:
-                    encrypted_cs.save(chap_json)
-                else:
-                    normal_cs.save(chap_json)
-                self.logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
-
-        normal_cs.close()
-        encrypted_cs.close()
-
-        self.saver.save(book_id)
-
-        self.logger.info(
-            "%s Novel '%s' download completed.",
-            TAG,
-            book_info.get("book_name", "unknown"),
-        )
-        return
-
-    def _handle_login(self) -> bool:
-        """
-        Perform login with automatic fallback to manual:
-
-        1. If manual_flag is False, try automatic login:
-           - On success, return True immediately.
-        2. Always attempt manual login if manual_flag is True.
-        3. Return True if manual login succeeds, False otherwise.
-        """
-        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
-
-        # First try automatic login
-        if not manual_flag and self._requester.login(manual_login=False):
-            return True
-
-        # try manual login
-        return self._requester.login(manual_login=True)
-
-
-def is_vip(html_str: str) -> bool:
-    """
-    Return True if page indicates VIP-only content.
-
-    :param html_str: Raw HTML string.
-    """
-    markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
-    return any(m in html_str for m in markers)

novel_downloader/core/downloaders/sfacg/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg
----------------------------------------
-
-"""
-
-from .sfacg_async import SfacgAsyncDownloader
-from .sfacg_sync import SfacgDownloader
-
-__all__ = [
-    "SfacgAsyncDownloader",
-    "SfacgDownloader",
-]

novel_downloader/core/downloaders/sfacg/sfacg_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg.sfacg_async
----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class SfacgAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/sfacg/sfacg_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.sfacg.sfacg_sync
---------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class SfacgDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "sfacg")

novel_downloader/core/downloaders/yamibo/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.yamibo
-----------------------------------------
-
-"""
-
-from .yamibo_async import YamiboAsyncDownloader
-from .yamibo_sync import YamiboDownloader
-
-__all__ = [
-    "YamiboAsyncDownloader",
-    "YamiboDownloader",
-]

novel_downloader/core/downloaders/yamibo/yamibo_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.yamibo.yamibo_async
------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class YamiboAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "yamibo")