novel-downloader 1.2.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

novel_downloader/core/downloaders/{base_downloader.py → base/base_sync.py}

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.base_downloader
--------------------------------------------------
+novel_downloader.core.downloaders.base.base_sync
+------------------------------------------------
 
 Defines the abstract base class `BaseDownloader`, which provides a
 common interface and reusable logic for all downloader implementations.
@@ -11,20 +10,17 @@ common interface and reusable logic for all downloader implementations.
 import abc
 import logging
 from pathlib import Path
-from typing import List
 
 from novel_downloader.config import DownloaderConfig
 from novel_downloader.core.interfaces import (
-    DownloaderProtocol,
     ParserProtocol,
-    RequesterProtocol,
     SaverProtocol,
+    SyncDownloaderProtocol,
+    SyncRequesterProtocol,
 )
 
-logger = logging.getLogger(__name__)
 
-
-class BaseDownloader(DownloaderProtocol, abc.ABC):
+class BaseDownloader(SyncDownloaderProtocol, abc.ABC):
     """
     Abstract downloader that defines the initialization interface
     and the general batch download flow.
@@ -34,10 +30,11 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
 
     def __init__(
         self,
-        requester: RequesterProtocol,
+        requester: SyncRequesterProtocol,
         parser: ParserProtocol,
         saver: SaverProtocol,
         config: DownloaderConfig,
+        site: str,
     ):
         """
         Initialize the downloader with its components.
@@ -51,23 +48,16 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         self._parser = parser
         self._saver = saver
         self._config = config
-        self._raw_data_dir = Path(config.raw_data_dir)
-        self._cache_dir = Path(config.cache_dir)
+        self._site = site
+
+        self._raw_data_dir = Path(config.raw_data_dir) / site
+        self._cache_dir = Path(config.cache_dir) / site
         self._raw_data_dir.mkdir(parents=True, exist_ok=True)
         self._cache_dir.mkdir(parents=True, exist_ok=True)
 
-    @abc.abstractmethod
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
-    def download(self, book_ids: List[str]) -> None:
+    def download(self, book_ids: list[str]) -> None:
         """
         The general batch download process:
           1. Iterate over all book IDs
@@ -75,41 +65,41 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
 
         :param book_ids: A list of book identifiers to download.
         """
+        self.prepare()
+
         for idx, book_id in enumerate(book_ids, start=1):
+            self.logger.debug(
+                "[downloader] Starting download for book_id: %s (%s/%s)",
+                book_id,
+                idx,
+                len(book_ids),
+            )
             try:
-                logger.debug(
-                    "[downloader] Starting download for book_id: %s (%s/%s)",
-                    book_id,
-                    idx,
-                    len(book_ids),
-                )
                 self.download_one(book_id)
             except Exception as e:
                 self._handle_download_exception(book_id, e)
 
-    def before_download(self, book_id: str) -> None:
+    @abc.abstractmethod
+    def download_one(self, book_id: str) -> None:
         """
-        Optional hook called before downloading each book.
+        The full download logic for a single book.
 
-        Subclasses can override this method to perform pre-download setup.
+        Subclasses must implement this method.
 
-        :param book_id: The book ID about to be processed.
+        :param book_id: The identifier of the book to download.
         """
-        pass
+        ...
 
-    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+    def prepare(self) -> None:
         """
-        Handle download errors in a consistent way.
-
-        This method can be overridden or extended to implement retry logic, etc.
+        Optional hook called before downloading each book.
 
-        :param book_id: The ID of the book that failed.
-        :param error: The exception raised during download.
+        Subclasses can override this method to perform pre-download setup.
         """
-        logger.warning("[downloader] Failed to download %s: %s", book_id, error)
+        return
 
     @property
-    def requester(self) -> RequesterProtocol:
+    def requester(self) -> SyncRequesterProtocol:
         """
         Access the current requester.
 
@@ -162,7 +152,27 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         return self._cache_dir
 
-    def set_requester(self, requester: RequesterProtocol) -> None:
+    @property
+    def site(self) -> str:
+        return self._site
+
+    @property
+    def save_html(self) -> bool:
+        return self._config.save_html
+
+    @property
+    def skip_existing(self) -> bool:
+        return self._config.skip_existing
+
+    @property
+    def login_required(self) -> bool:
+        return self._config.login_required
+
+    @property
+    def request_interval(self) -> float:
+        return self._config.request_interval
+
+    def set_requester(self, requester: SyncRequesterProtocol) -> None:
         """
         Replace the requester instance with a new one.
 
@@ -185,3 +195,14 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param saver: The new saver to be used.
         """
         self._saver = saver
+
+    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
+        """
+        Handle download errors in a consistent way.
+
+        This method can be overridden or extended to implement retry logic, etc.
+
+        :param book_id: The ID of the book that failed.
+        :param error: The exception raised during download.
+        """
+        self.logger.warning("[downloader] Failed to download %s: %s", book_id, error)

novel_downloader/core/downloaders/biquge/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.biquge
+----------------------------------------
+
+"""
+
+from .biquge_sync import BiqugeDownloader
+
+__all__ = [
+    "BiqugeDownloader",
+]

novel_downloader/core/downloaders/biquge/biquge_sync.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.biquge.biquge_sync
+----------------------------------------------------
+
+"""
+
+from novel_downloader.config.models import DownloaderConfig
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces.parser import ParserProtocol
+from novel_downloader.core.interfaces.saver import SaverProtocol
+from novel_downloader.core.interfaces.sync_requester import SyncRequesterProtocol
+
+
+class BiqugeDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        requester: SyncRequesterProtocol,
+        parser: ParserProtocol,
+        saver: SaverProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/common/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.common
+----------------------------------------
+
+"""
+
+from .common_async import CommonAsyncDownloader
+from .common_sync import CommonDownloader
+
+__all__ = [
+    "CommonAsyncDownloader",
+    "CommonDownloader",
+]

novel_downloader/core/downloaders/{common_asynb_downloader.py → common/common_async.py}

@@ -1,30 +1,28 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.common_asynb_downloader
----------------------------------------------------------
+novel_downloader.core.downloaders.common.common_async
+-----------------------------------------------------
 
-This module defines `CommonAsynbDownloader`.
 """
 
 import asyncio
 import json
 import logging
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
-from typing import Any, Dict, Tuple
+from typing import Any
 
 from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.downloaders.base import BaseAsyncDownloader
 from novel_downloader.core.interfaces import (
     AsyncRequesterProtocol,
     ParserProtocol,
     SaverProtocol,
 )
+from novel_downloader.utils.chapter_storage import ChapterDict, ChapterStorage
 from novel_downloader.utils.file_utils import save_as_json, save_as_txt
 from novel_downloader.utils.network import download_image_as_bytes
 from novel_downloader.utils.time_utils import calculate_time_difference
 
-from .base_async_downloader import BaseAsyncDownloader
-
 logger = logging.getLogger(__name__)
 
 
@@ -64,19 +62,25 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
         assert isinstance(self.requester, AsyncRequesterProtocol)
 
         TAG = "[AsyncDownloader]"
+        wait_time = self.config.request_interval
+
         raw_base = self.raw_data_dir / book_id
         cache_base = self.cache_dir / book_id
         info_path = raw_base / "book_info.json"
         chapters_html_dir = cache_base / "html"
-        chapter_dir = raw_base / "chapters"
 
         raw_base.mkdir(parents=True, exist_ok=True)
-        chapter_dir.mkdir(parents=True, exist_ok=True)
         if self.save_html:
             chapters_html_dir.mkdir(parents=True, exist_ok=True)
+        normal_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
 
         # load or fetch book_info
-        book_info: Dict[str, Any]
+        book_info: dict[str, Any]
         re_fetch = True
         if info_path.exists():
             try:
@@ -89,9 +93,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
                 re_fetch = True
 
         if re_fetch:
-            info_html = await self.requester.get_book_info(
-                book_id, self.request_interval
-            )
+            info_html = await self.requester.get_book_info(book_id)
             if self.save_html:
                 save_as_txt(info_html, chapters_html_dir / "info.html")
             book_info = self.parser.parse_book_info(info_html)
@@ -99,6 +101,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
                 save_as_json(book_info, info_path)
             else:
                 logger.warning("%s 书籍信息未找到, book_id = %s", TAG, book_id)
+            await asyncio.sleep(wait_time)
         else:
             book_info = json.loads(info_path.read_text("utf-8"))
 
@@ -111,7 +114,8 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
 
         # setup queue, semaphore, executor
         semaphore = asyncio.Semaphore(self.download_workers)
-        queue: asyncio.Queue[Tuple[str, str]] = asyncio.Queue()
+        queue: asyncio.Queue[tuple[str, str]] = asyncio.Queue()
+        save_queue: asyncio.Queue[ChapterDict] = asyncio.Queue()
         loop = asyncio.get_running_loop()
         executor = (
             ProcessPoolExecutor() if self.use_process_pool else ThreadPoolExecutor()
@@ -125,12 +129,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
                         executor, self.parser.parse_chapter, html, cid
                     )
                     if chap_json:
-                        await loop.run_in_executor(
-                            executor,
-                            save_as_json,
-                            chap_json,
-                            chapter_dir / f"{cid}.json",
-                        )
+                        await save_queue.put(chap_json)
                         logger.info(
                             "%s [Parser-%d] saved chapter %s", TAG, worker_id, cid
                         )
@@ -141,27 +140,34 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
                 finally:
                     queue.task_done()
 
-        async def download_worker(chap: Dict[str, Any]) -> None:
+        async def saver_loop(
+            cs: ChapterStorage,
+            queue: asyncio.Queue[ChapterDict],
+        ) -> None:
+            while True:
+                data = await queue.get()
+                try:
+                    cs.save(data)
+                except Exception as e:
+                    logger.error(
+                        "[saver] Error saving chapter %s: %s",
+                        data.get("id"),
+                        e,
+                    )
+                finally:
+                    queue.task_done()
+
+        async def download_worker(chap: dict[str, Any]) -> None:
             cid = str(chap.get("chapterId") or "")
             if not cid:
                 return
-            target = chapter_dir / f"{cid}.json"
-            if target.exists() and self.skip_existing:
+            if normal_cs.exists(cid) and self.skip_existing:
                 logger.info("%s skipping existing chapter %s", TAG, cid)
                 return
 
             try:
                 async with semaphore:
-                    html = await self.requester.get_book_chapter(
-                        book_id, cid, self.request_interval
-                    )
-                if self.save_html:
-                    await loop.run_in_executor(
-                        executor,
-                        save_as_txt,
-                        html,
-                        chapters_html_dir / f"{cid}.html",
-                    )
+                    html = await self.requester.get_book_chapter(book_id, cid)
                 await queue.put((cid, html))
                 logger.info("%s downloaded chapter %s", TAG, cid)
             except Exception as e:
@@ -171,6 +177,7 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
         parsers = [
             asyncio.create_task(parser_worker(i)) for i in range(self.parser_workers)
         ]
+        chapter_saver = asyncio.create_task(saver_loop(normal_cs, save_queue))
 
         # enqueue + run downloads
         download_tasks = []
@@ -180,8 +187,10 @@ class CommonAsyncDownloader(BaseAsyncDownloader):
 
         await asyncio.gather(*download_tasks)
         await queue.join()  # wait until all parsed
+        await save_queue.join()
         for p in parsers:
             p.cancel()  # stop parser loops
+        chapter_saver.cancel()
 
         # final save
         await loop.run_in_executor(executor, self.saver.save, book_id)

novel_downloader/core/downloaders/{common_downloader.py → common/common_sync.py}

@@ -1,27 +1,29 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.common_downloader
----------------------------------------------------
+novel_downloader.core.downloaders.common.common_sync
+----------------------------------------------------
 
 This module defines `CommonDownloader`.
 """
 
 import json
 import logging
-from typing import Any, Dict
+from typing import Any
 
 from novel_downloader.config import DownloaderConfig
+from novel_downloader.core.downloaders.base import BaseDownloader
 from novel_downloader.core.interfaces import (
     ParserProtocol,
-    RequesterProtocol,
     SaverProtocol,
+    SyncRequesterProtocol,
 )
+from novel_downloader.utils.chapter_storage import ChapterStorage
 from novel_downloader.utils.file_utils import save_as_json, save_as_txt
 from novel_downloader.utils.network import download_image_as_bytes
-from novel_downloader.utils.time_utils import calculate_time_difference
-
-from .base_downloader import BaseDownloader
+from novel_downloader.utils.time_utils import (
+    calculate_time_difference,
+    sleep_with_random_delay,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -33,7 +35,7 @@ class CommonDownloader(BaseDownloader):
 
     def __init__(
         self,
-        requester: RequesterProtocol,
+        requester: SyncRequesterProtocol,
         parser: ParserProtocol,
         saver: SaverProtocol,
         config: DownloaderConfig,
@@ -48,7 +50,7 @@ class CommonDownloader(BaseDownloader):
         :param config: Downloader configuration object.
         :param site: Identifier for the site the downloader is targeting.
         """
-        super().__init__(requester, parser, saver, config)
+        super().__init__(requester, parser, saver, config, site)
         self._site = site
 
     def download_one(self, book_id: str) -> None:
@@ -60,19 +62,24 @@ class CommonDownloader(BaseDownloader):
         TAG = "[Downloader]"
         save_html = self.config.save_html
         skip_existing = self.config.skip_existing
-        site = self.site
         wait_time = self.config.request_interval
 
-        raw_base = self.raw_data_dir / site / book_id
-        cache_base = self.cache_dir / site / book_id
+        raw_base = self.raw_data_dir / book_id
+        cache_base = self.cache_dir / book_id
         info_path = raw_base / "book_info.json"
-        chapter_dir = raw_base / "chapters"
         chapters_html_dir = cache_base / "html"
 
         raw_base.mkdir(parents=True, exist_ok=True)
-        chapter_dir.mkdir(parents=True, exist_ok=True)
+        if self.save_html:
+            chapters_html_dir.mkdir(parents=True, exist_ok=True)
+        normal_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
 
-        book_info: Dict[str, Any]
+        book_info: dict[str, Any]
 
         try:
             if not info_path.exists():
@@ -87,7 +94,7 @@ class CommonDownloader(BaseDownloader):
             if days > 1:
                 raise FileNotFoundError  # trigger re-fetch
         except Exception:
-            info_html = self.requester.get_book_info(book_id, wait_time)
+            info_html = self.requester.get_book_info(book_id)
             if save_html:
                 info_html_path = chapters_html_dir / "info.html"
                 save_as_txt(info_html, info_html_path)
@@ -97,6 +104,7 @@ class CommonDownloader(BaseDownloader):
                 and book_info.get("update_time", "") != "未找到更新时间"
             ):
                 save_as_json(book_info, info_path)
+            sleep_with_random_delay(wait_time, mul_spread=1.1, max_sleep=wait_time + 2)
 
         # download cover
         cover_url = book_info.get("cover_url", "")
@@ -116,8 +124,7 @@ class CommonDownloader(BaseDownloader):
                     logger.warning("%s Skipping chapter without chapterId", TAG)
                     continue
 
-                chap_path = chapter_dir / f"{cid}.json"
-                if chap_path.exists() and skip_existing:
+                if normal_cs.exists(cid) and skip_existing:
                     logger.debug(
                         "%s Chapter already exists, skipping: %s",
                         TAG,
@@ -128,7 +135,7 @@ class CommonDownloader(BaseDownloader):
                 chap_title = chap.get("title", "")
                 logger.info("%s Fetching chapter: %s (%s)", TAG, chap_title, cid)
                 try:
-                    chap_html = self.requester.get_book_chapter(book_id, cid, wait_time)
+                    chap_html = self.requester.get_book_chapter(book_id, cid)
 
                     if save_html:
                         html_path = chapters_html_dir / f"{cid}.html"
@@ -141,6 +148,10 @@ class CommonDownloader(BaseDownloader):
                         )
 
                     chap_json = self.parser.parse_chapter(chap_html, cid)
+
+                    sleep_with_random_delay(
+                        wait_time, mul_spread=1.1, max_sleep=wait_time + 2
+                    )
                     if not chap_json:
                         logger.warning(
                             "%s Parsed chapter json is empty, skipping: %s (%s)",
@@ -159,9 +170,10 @@ class CommonDownloader(BaseDownloader):
                     )
                     continue
 
-                save_as_json(chap_json, chap_path)
+                normal_cs.save(chap_json)
                 logger.info("%s Saved chapter: %s (%s)", TAG, chap_title, cid)
 
+        normal_cs.close()
         self.saver.save(book_id)
 
         logger.info(

novel_downloader/core/downloaders/qidian/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qidian
+----------------------------------------
+
+"""
+
+from .qidian_sync import QidianDownloader
+
+__all__ = ["QidianDownloader"]