novel-downloader 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

novel_downloader/core/downloaders/{qidian_downloader.py → qidian/qidian_sync.py}

@@ -1,31 +1,30 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.qidian_downloader
----------------------------------------------------
+novel_downloader.core.downloaders.qidian.qidian_sync
+----------------------------------------------------
 
 This module defines `QidianDownloader`, a platform-specific downloader
 implementation for retrieving novels from Qidian (起点中文网).
 """
 
 import json
-import logging
-from typing import Any, Dict
+from typing import Any
 
 from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.downloaders.base import BaseDownloader
 from novel_downloader.core.interfaces import (
     ParserProtocol,
-    RequesterProtocol,
     SaverProtocol,
+    SyncRequesterProtocol,
 )
+from novel_downloader.utils.chapter_storage import ChapterStorage
 from novel_downloader.utils.file_utils import save_as_json, save_as_txt
 from novel_downloader.utils.network import download_image_as_bytes
 from novel_downloader.utils.state import state_mgr
-from novel_downloader.utils.time_utils import calculate_time_difference
-
-from .base_downloader import BaseDownloader
-
-logger = logging.getLogger(__name__)
+from novel_downloader.utils.time_utils import (
+    calculate_time_difference,
+    sleep_with_random_delay,
+)
 
 
 class QidianDownloader(BaseDownloader):
@@ -35,36 +34,17 @@ class QidianDownloader(BaseDownloader):
 
     def __init__(
         self,
-        requester: RequesterProtocol,
+        requester: SyncRequesterProtocol,
         parser: ParserProtocol,
         saver: SaverProtocol,
         config: DownloaderConfig,
     ):
-        super().__init__(requester, parser, saver, config)
+        super().__init__(requester, parser, saver, config, "qidian")
 
         self._site_key = "qidian"
         self._is_logged_in = self._handle_login()
         state_mgr.set_manual_login_flag(self._site_key, not self._is_logged_in)
 
-    def _handle_login(self) -> bool:
-        """
-        Perform login with automatic fallback to manual:
-
-        1. If manual_flag is False, try automatic login:
-           - On success, return True immediately.
-        2. Always attempt manual login if manual_flag is True.
-        3. Return True if manual login succeeds, False otherwise.
-        """
-        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
-
-        # First try automatic login
-        if not manual_flag:
-            if self._requester.login(manual_login=False):
-                return True
-
-        # try manual login
-        return self._requester.login(manual_login=True)
-
     def download_one(self, book_id: str) -> None:
         """
         The full download logic for a single book.
@@ -72,8 +52,10 @@ class QidianDownloader(BaseDownloader):
         :param book_id: The identifier of the book to download.
         """
         if not self._is_logged_in:
-            logger.warning(
-                f"[{self._site_key}] login failed, skipping download of {book_id}"
+            self.logger.warning(
+                "[%s] login failed, skipping download of %s",
+                self._site_key,
+                book_id,
             )
             return
 
@@ -81,20 +63,28 @@ class QidianDownloader(BaseDownloader):
         save_html = self.config.save_html
         skip_existing = self.config.skip_existing
         wait_time = self.config.request_interval
+        scroll = self.config.mode == "browser"
 
-        raw_base = self.raw_data_dir / "qidian" / book_id
-        cache_base = self.cache_dir / "qidian" / book_id
+        raw_base = self.raw_data_dir / book_id
+        cache_base = self.cache_dir / book_id
         info_path = raw_base / "book_info.json"
-        chapter_dir = raw_base / "chapters"
-        encrypted_chapter_dir = raw_base / "encrypted_chapters"
-        if save_html:
-            chapters_html_dir = cache_base / "html"
+        chapters_html_dir = cache_base / "html"
 
         raw_base.mkdir(parents=True, exist_ok=True)
-        chapter_dir.mkdir(parents=True, exist_ok=True)
-        encrypted_chapter_dir.mkdir(parents=True, exist_ok=True)
+        normal_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
+        encrypted_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="encrypted_chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
 
-        book_info: Dict[str, Any]
+        book_info: dict[str, Any]
 
         try:
             if not info_path.exists():
@@ -103,13 +93,13 @@ class QidianDownloader(BaseDownloader):
             days, hrs, mins, secs = calculate_time_difference(
                 book_info.get("update_time", ""), "UTC+8"
             )
-            logger.info(
+            self.logger.info(
                 "%s Last updated %dd %dh %dm %ds ago", TAG, days, hrs, mins, secs
             )
             if days > 1:
                 raise FileNotFoundError  # trigger re-fetch
         except Exception:
-            info_html = self.requester.get_book_info(book_id, wait_time)
+            info_html = self.requester.get_book_info(book_id)
             if save_html:
                 info_html_path = chapters_html_dir / "info.html"
                 save_as_txt(info_html, info_html_path)
@@ -119,29 +109,28 @@ class QidianDownloader(BaseDownloader):
                 and book_info.get("update_time", "") != "未找到更新时间"
             ):
                 save_as_json(book_info, info_path)
+            sleep_with_random_delay(wait_time, mul_spread=1.1, max_sleep=wait_time + 2)
 
         # download cover
         cover_url = book_info.get("cover_url", "")
         if cover_url:
             cover_bytes = download_image_as_bytes(cover_url, raw_base)
             if not cover_bytes:
-                logger.warning("%s Failed to download cover: %s", TAG, cover_url)
+                self.logger.warning("%s Failed to download cover: %s", TAG, cover_url)
 
         # enqueue chapters
         for vol in book_info.get("volumes", []):
             vol_name = vol.get("volume_name", "")
-            logger.info("%s Enqueuing volume: %s", TAG, vol_name)
+            self.logger.info("%s Enqueuing volume: %s", TAG, vol_name)
 
             for chap in vol.get("chapters", []):
                 cid = chap.get("chapterId")
                 if not cid:
-                    logger.warning("%s Skipping chapter without chapterId", TAG)
+                    self.logger.warning("%s Skipping chapter without chapterId", TAG)
                     continue
 
-                chap_path = chapter_dir / f"{cid}.json"
-
-                if chap_path.exists() and skip_existing:
-                    logger.debug(
+                if normal_cs.exists(cid) and skip_existing:
+                    self.logger.debug(
                         "%s Chapter already exists, skipping: %s",
                         TAG,
                         cid,
@@ -149,16 +138,19 @@ class QidianDownloader(BaseDownloader):
                     continue
 
                 chap_title = chap.get("title", "")
-                logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
-                chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+                self.logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
+                chap_html = self.requester.get_book_chapter(book_id, cid)
+                if scroll:
+                    self.requester.scroll_page(wait_time * 2)  # type: ignore[attr-defined]
+                else:
+                    sleep_with_random_delay(
+                        wait_time, mul_spread=1.1, max_sleep=wait_time + 2
+                    )
 
                 is_encrypted = self.parser.is_encrypted(chap_html)  # type: ignore[attr-defined]
 
-                folder = encrypted_chapter_dir if is_encrypted else chapter_dir
-                chap_path = folder / f"{cid}.json"
-
-                if chap_path.exists() and skip_existing:
-                    logger.debug(
+                if is_encrypted and encrypted_cs.exists(cid) and skip_existing:
+                    self.logger.debug(
                         "%s Chapter already exists, skipping: %s",
                         TAG,
                         cid,
@@ -171,13 +163,13 @@ class QidianDownloader(BaseDownloader):
                     )
                     html_path = folder / f"{cid}.html"
                     save_as_txt(chap_html, html_path, on_exist="skip")
-                    logger.debug(
+                    self.logger.debug(
                         "%s Saved raw HTML for chapter %s to %s", TAG, cid, html_path
                     )
 
                 chap_json = self.parser.parse_chapter(chap_html, cid)
                 if not chap_json:
-                    logger.warning(
+                    self.logger.warning(
                         "%s Parsed chapter json is empty, skipping: %s (%s)",
                         TAG,
                         chap_title,
@@ -185,18 +177,42 @@ class QidianDownloader(BaseDownloader):
                     )
                     continue
 
-                save_as_json(chap_json, chap_path)
-                logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+                if is_encrypted:
+                    encrypted_cs.save(chap_json)
+                else:
+                    normal_cs.save(chap_json)
+                self.logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
+
+        normal_cs.close()
+        encrypted_cs.close()
 
         self.saver.save(book_id)
 
-        logger.info(
+        self.logger.info(
             "%s Novel '%s' download completed.",
             TAG,
             book_info.get("book_name", "unknown"),
         )
         return
 
+    def _handle_login(self) -> bool:
+        """
+        Perform login with automatic fallback to manual:
+
+        1. If manual_flag is False, try automatic login:
+           - On success, return True immediately.
+        2. Always attempt manual login if manual_flag is True.
+        3. Return True if manual login succeeds, False otherwise.
+        """
+        manual_flag = state_mgr.get_manual_login_flag(self._site_key)
+
+        # First try automatic login
+        if not manual_flag and self._requester.login(manual_login=False):
+            return True
+
+        # try manual login
+        return self._requester.login(manual_login=True)
+
 
 def is_vip(html_str: str) -> bool:
     """

novel_downloader/core/factory/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.factory
 -----------------------------
@@ -8,18 +7,18 @@ This package provides factory methods for dynamically retrieving components
 based on runtime parameters such as site name or content type.
 """
 
-from .downloader_factory import (
+from .downloader import (
     get_async_downloader,
     get_downloader,
     get_sync_downloader,
 )
-from .parser_factory import get_parser
-from .requester_factory import (
+from .parser import get_parser
+from .requester import (
     get_async_requester,
     get_requester,
     get_sync_requester,
 )
-from .saver_factory import get_saver
+from .saver import get_saver
 
 __all__ = [
     "get_async_downloader",

novel_downloader/core/factory/{downloader_factory.py → downloader.py}

@@ -1,23 +1,17 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.factory.downloader_factory
 ------------------------------------------------
 
 This module implements a factory function for creating downloader instances
 based on the site name and parser mode specified in the configuration.
-
-- get_async_downloader -> always returns a CommonAsyncDownloader
-- get_sync_downloader  -> returns a site-specific downloader or CommonDownloader
-- get_downloader       -> dispatches to one of the above based on config.mode
-
-To add support for new sites or modes, extend the `_site_map` accordingly.
 """
 
-from typing import Union
+from typing import cast
 
 from novel_downloader.config import DownloaderConfig, load_site_rules
 from novel_downloader.core.downloaders import (
+    BiqugeDownloader,
     CommonAsyncDownloader,
     CommonDownloader,
     QidianDownloader,
@@ -25,15 +19,18 @@ from novel_downloader.core.downloaders import (
 from novel_downloader.core.interfaces import (
     AsyncDownloaderProtocol,
     AsyncRequesterProtocol,
-    DownloaderProtocol,
     ParserProtocol,
-    RequesterProtocol,
     SaverProtocol,
+    SyncDownloaderProtocol,
+    SyncRequesterProtocol,
 )
 
-_site_map = {
+# _async_site_map = {
+#     # "biquge": ...
+# }
+_sync_site_map = {
     "qidian": QidianDownloader,
-    # "biquge": ...
+    "biquge": BiqugeDownloader,
 }
 
 
@@ -56,14 +53,18 @@ def get_async_downloader(
     :return: An instance of a downloader class
 
     :raises ValueError: If a site-specific downloader does not support async mode.
-    :raises TypeError: If the provided requester does not match the required protocol
-                    for the chosen mode (sync vs async).
+    :raises TypeError: If the provided requester does not match the required protocol.
     """
     site_key = site.lower()
 
     if not isinstance(requester, AsyncRequesterProtocol):
         raise TypeError("Async mode requires an AsyncRequesterProtocol")
 
+    # site-specific
+    # if site_key in _async_site_map:
+    #     return _async_site_map[site_key](requester, parser, saver, config)
+
+    # fallback
     site_rules = load_site_rules()
     site_rule = site_rules.get(site_key)
     if site_rule is None:
@@ -73,12 +74,12 @@ def get_async_downloader(
 
 
 def get_sync_downloader(
-    requester: RequesterProtocol,
+    requester: SyncRequesterProtocol,
     parser: ParserProtocol,
     saver: SaverProtocol,
     site: str,
     config: DownloaderConfig,
-) -> DownloaderProtocol:
+) -> SyncDownloaderProtocol:
     """
     Returns a DownloaderProtocol for the given site.
     First tries a site-specific downloader (e.g. QidianDownloader),
@@ -93,17 +94,16 @@ def get_sync_downloader(
     :return: An instance of a downloader class
 
     :raises ValueError: If a site-specific downloader does not support async mode.
-    :raises TypeError: If the provided requester does not match the required protocol
-                    for the chosen mode (sync vs async).
+    :raises TypeError: If the provided requester does not match the required protocol.
     """
     site_key = site.lower()
 
-    if not isinstance(requester, RequesterProtocol):
+    if not isinstance(requester, SyncRequesterProtocol):
         raise TypeError("Sync mode requires a RequesterProtocol")
 
     # site-specific
-    if site_key in _site_map:
-        return _site_map[site_key](requester, parser, saver, config)
+    if site_key in _sync_site_map:
+        return _sync_site_map[site_key](requester, parser, saver, config)
 
     # fallback
     site_rules = load_site_rules()
@@ -115,12 +115,12 @@ def get_sync_downloader(
 
 
 def get_downloader(
-    requester: Union[AsyncRequesterProtocol, RequesterProtocol],
+    requester: AsyncRequesterProtocol | SyncRequesterProtocol,
     parser: ParserProtocol,
     saver: SaverProtocol,
     site: str,
     config: DownloaderConfig,
-) -> Union[AsyncDownloaderProtocol, DownloaderProtocol]:
+) -> AsyncDownloaderProtocol | SyncDownloaderProtocol:
     """
     Dispatches to get_async_downloader if config.mode == 'async',
     otherwise to get_sync_downloader.
@@ -134,16 +134,17 @@ def get_downloader(
     :return: An instance of a downloader class
 
     :raises ValueError: If a site-specific downloader does not support async mode.
-    :raises TypeError: If the provided requester does not match the required protocol
-                    for the chosen mode (sync vs async).
+    :raises TypeError: If the provided requester does not match the required protocol.
     """
-    mode = config.mode.lower()
-    if mode == "async":
-        if not isinstance(requester, AsyncRequesterProtocol):
-            raise TypeError("Async mode requires an AsyncRequesterProtocol")
-        return get_async_downloader(requester, parser, saver, site, config)
-    if mode in ("browser", "session"):
-        if not isinstance(requester, RequesterProtocol):
-            raise TypeError("Sync mode requires a RequesterProtocol")
-        return get_sync_downloader(requester, parser, saver, site, config)
-    raise ValueError(f"Unknown mode '{config.mode}' for site '{site}'")
+    if requester.is_async():
+        if config.mode.lower() != "async":
+            raise TypeError("Requester is async, but config.mode is not 'async'")
+        async_requester = cast(AsyncRequesterProtocol, requester)
+        return get_async_downloader(async_requester, parser, saver, site, config)
+    else:
+        if config.mode.lower() not in ("browser", "session"):
+            raise TypeError(
+                "Requester is sync, but config.mode is not 'browser' or 'session'"
+            )
+        sync_requester = cast(SyncRequesterProtocol, requester)
+        return get_sync_downloader(sync_requester, parser, saver, site, config)

novel_downloader/core/factory/{parser_factory.py → parser.py}

@@ -1,35 +1,33 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.factory.parser_factory
 --------------------------------------------
 
 This module implements a factory function for creating parser instances
 based on the site name and parser mode specified in the configuration.
-
-Currently supported:
-- Site: 'qidian'
-    - Modes:
-        - 'browser': QidianBrowserParser
-        - 'session': (Not implemented yet)
-
-To add support for new sites or modes, extend the `_site_map` accordingly.
 """
 
+from collections.abc import Callable
+
 from novel_downloader.config import ParserConfig, load_site_rules
 from novel_downloader.core.interfaces import ParserProtocol
 from novel_downloader.core.parsers import (
+    BiqugeParser,
     CommonParser,
     QidianBrowserParser,
     QidianSessionParser,
 )
 
-_site_map = {
+ParserBuilder = Callable[[ParserConfig], ParserProtocol]
+
+_site_map: dict[str, dict[str, ParserBuilder]] = {
     "qidian": {
         "browser": QidianBrowserParser,
         "session": QidianSessionParser,
     },
-    # "biquge": ...
+    "biquge": {
+        "session": BiqugeParser,
+    },
 }
 
 
@@ -47,11 +45,11 @@ def get_parser(site: str, config: ParserConfig) -> ParserProtocol:
         site_entry = _site_map[site_key]
         if isinstance(site_entry, dict):
             parser_class = site_entry.get(config.mode)
-            if parser_class is None:
-                raise ValueError(f"Unsupported mode '{config.mode}' for site '{site}'")
         else:
             parser_class = site_entry
-        return parser_class(config)
+
+        if parser_class:
+            return parser_class(config)
 
     # Fallback: site not mapped specially, try to load rule
     site_rules = load_site_rules()

novel_downloader/core/factory/{requester_factory.py → requester.py}

@@ -1,38 +1,45 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.factory.requester_factory
 -----------------------------------------------
 
 This module implements a factory function for retrieving requester instances
 based on the target novel platform (site).
-
-- get_async_requester -> returns AsyncRequesterProtocol
-- get_sync_requester  -> returns RequesterProtocol
-- get_requester       -> dispatches to one of the above based on config.mode
-
-To add support for new sites or modes, extend the `_site_map` accordingly.
 """
 
-from typing import Callable, Union
+from collections.abc import Callable
 
 from novel_downloader.config import RequesterConfig, load_site_rules
-from novel_downloader.core.interfaces import AsyncRequesterProtocol, RequesterProtocol
+from novel_downloader.core.interfaces import (
+    AsyncRequesterProtocol,
+    SyncRequesterProtocol,
+)
 from novel_downloader.core.requesters import (
+    BiqugeSession,
     CommonAsyncSession,
     CommonSession,
     QidianBrowser,
     QidianSession,
 )
 
-_site_map: dict[
+AsyncRequesterBuilder = Callable[[RequesterConfig], AsyncRequesterProtocol]
+SyncRequesterBuilder = Callable[[RequesterConfig], SyncRequesterProtocol]
+
+
+# _async_site_map: dict[str, AsyncRequesterBuilder] = {
+#     # "biquge": ...
+# }
+_sync_site_map: dict[
     str,
-    dict[str, Callable[[RequesterConfig], RequesterProtocol]],
+    dict[str, SyncRequesterBuilder],
 ] = {
     "qidian": {
         "session": QidianSession,
         "browser": QidianBrowser,
     },
+    "biquge": {
+        "session": BiqugeSession,
+    },
 }
 
 
@@ -48,6 +55,12 @@ def get_async_requester(
     :return: An instance of a requester class
     """
     site_key = site.lower()
+
+    # site-specific
+    # if site_key in _async_site_map:
+    #     return _async_site_map[site_key](config)
+
+    # fallback
     site_rules = load_site_rules()
     site_rule = site_rules.get(site_key)
     if site_rule is None:
@@ -59,7 +72,7 @@ def get_async_requester(
 def get_sync_requester(
     site: str,
     config: RequesterConfig,
-) -> RequesterProtocol:
+) -> SyncRequesterProtocol:
     """
     Returns a RequesterProtocol for the given site.
 
@@ -68,15 +81,15 @@ def get_sync_requester(
     :return: An instance of a requester class
     """
     site_key = site.lower()
-    site_entry = _site_map.get(site_key)
+    site_entry = _sync_site_map.get(site_key)
 
-    # site-specific implementation for this mode
+    # site-specific
     if site_entry:
         cls = site_entry.get(config.mode)
         if cls:
             return cls(config)
 
-    # fallback to CommonSession
+    # fallback
     site_rules = load_site_rules()
     site_rule = site_rules.get(site_key)
     if site_rule is None:
@@ -88,7 +101,7 @@ def get_sync_requester(
 def get_requester(
     site: str,
     config: RequesterConfig,
-) -> Union[AsyncRequesterProtocol, RequesterProtocol]:
+) -> AsyncRequesterProtocol | SyncRequesterProtocol:
     """
     Dispatches to either get_async_requester or get_sync_requester
     based on config.mode. Treats 'browser' and 'async' as async modes,

novel_downloader/core/factory/{saver_factory.py → saver.py}

@@ -1,29 +1,23 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.factory.parser_factory
 --------------------------------------------
 
 This module implements a factory function for creating saver instances
 based on the site name and parser mode specified in the configuration.
-
-Currently supported:
-- Site: 'qidian'
-    - QidianSaver
-
-To add support for new sites or modes, extend the `_site_map` accordingly.
 """
 
 from novel_downloader.config import SaverConfig, load_site_rules
 from novel_downloader.core.interfaces import SaverProtocol
 from novel_downloader.core.savers import (
+    BiqugeSaver,
     CommonSaver,
     QidianSaver,
 )
 
 _site_map = {
     "qidian": QidianSaver,
-    # "biquge": ...
+    "biquge": BiqugeSaver,
 }
 
 
@@ -37,11 +31,12 @@ def get_saver(site: str, config: SaverConfig) -> SaverProtocol:
     """
     site_key = site.lower()
 
+    # site-specific
     saver_class = _site_map.get(site_key)
     if saver_class:
         return saver_class(config)
 
-    # Fallback: check site_rules
+    # Fallback
     site_rules = load_site_rules()
     if site_key not in site_rules:
         raise ValueError(f"Unsupported site: {site}")