novel-downloader 1.3.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/core/downloaders/qidian.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qidian
+----------------------------------------
+
+"""
+
+import asyncio
+import json
+from collections.abc import Awaitable, Callable
+from contextlib import suppress
+from typing import Any, cast
+
+from novel_downloader.core.downloaders.base import BaseDownloader
+from novel_downloader.core.interfaces import (
+    ExporterProtocol,
+    FetcherProtocol,
+    ParserProtocol,
+)
+from novel_downloader.models import (
+    ChapterDict,
+    CidTask,
+    DownloaderConfig,
+    HtmlTask,
+)
+from novel_downloader.utils.chapter_storage import ChapterStorage
+from novel_downloader.utils.file_utils import save_as_json, save_as_txt
+from novel_downloader.utils.time_utils import (
+    async_sleep_with_random_delay,
+    calculate_time_difference,
+)
+
+
+class QidianDownloader(BaseDownloader):
+    """
+    Specialized downloader for Qidian novels.
+    """
+
+    def __init__(
+        self,
+        fetcher: FetcherProtocol,
+        parser: ParserProtocol,
+        exporter: ExporterProtocol,
+        config: DownloaderConfig,
+    ):
+        config.request_interval = max(1.0, config.request_interval)
+        super().__init__(fetcher, parser, exporter, config, "qidian")
+
+    async def _download_one(
+        self,
+        book_id: str,
+        *,
+        progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book_id: The identifier of the book to download.
+        """
+        TAG = "[Downloader]"
+
+        raw_base = self.raw_data_dir / book_id
+        cache_base = self.cache_dir / book_id
+        info_path = raw_base / "book_info.json"
+        chapters_html_dir = cache_base / "html"
+
+        raw_base.mkdir(parents=True, exist_ok=True)
+        if self.save_html:
+            chapters_html_dir.mkdir(parents=True, exist_ok=True)
+        normal_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
+        encrypted_cs = ChapterStorage(
+            raw_base=raw_base,
+            namespace="encrypted_chapters",
+            backend_type=self._config.storage_backend,
+            batch_size=self._config.storage_batch_size,
+        )
+
+        # load or fetch book_info
+        book_info: dict[str, Any]
+        re_fetch = True
+        old_data: dict[str, Any] = {}
+
+        if info_path.exists():
+            try:
+                old_data = json.loads(info_path.read_text("utf-8"))
+                days, *_ = calculate_time_difference(
+                    old_data.get("update_time", ""), "UTC+8"
+                )
+                re_fetch = days > 1
+            except Exception:
+                re_fetch = True
+
+        if re_fetch:
+            info_html = await self.fetcher.get_book_info(book_id)
+            if self.save_html:
+                for i, html in enumerate(info_html):
+                    save_as_txt(html, chapters_html_dir / f"info_{i}.html")
+            book_info = self.parser.parse_book_info(info_html)
+
+            if book_info.get("book_name") != "未找到书名":
+                save_as_json(book_info, info_path)
+            else:
+                self.logger.warning("%s 书籍信息未找到, book_id = %s", TAG, book_id)
+                book_info = old_data or {"book_name": "未找到书名"}
+        else:
+            book_info = old_data
+
+        vols = book_info.get("volumes", [])
+        total_chapters = 0
+        for vol in vols:
+            total_chapters += len(vol.get("chapters", []))
+        if total_chapters == 0:
+            self.logger.warning("%s 书籍没有章节可下载: book_id=%s", TAG, book_id)
+            return
+
+        completed_count = 0
+
+        # setup queue
+        cid_queue: asyncio.Queue[CidTask] = asyncio.Queue()
+        html_queue: asyncio.Queue[HtmlTask] = asyncio.Queue()
+        save_queue: asyncio.Queue[ChapterDict] = asyncio.Queue()
+
+        async def fetcher_worker(
+            book_id: str,
+            cid_queue: asyncio.Queue[CidTask],
+            html_queue: asyncio.Queue[HtmlTask],
+            retry_times: int,
+        ) -> None:
+            while True:
+                task = await cid_queue.get()
+                cid = task.cid
+                if not cid:
+                    self.logger.warning("[Fetcher] Skipped empty cid task: %s", task)
+                    cid_queue.task_done()
+                    continue
+
+                try:
+                    html_list = await self.fetcher.get_book_chapter(book_id, cid)
+                    await html_queue.put(
+                        HtmlTask(cid=cid, retry=task.retry, html_list=html_list)
+                    )
+                    self.logger.info("[Fetcher] Downloaded chapter %s", cid)
+                    await async_sleep_with_random_delay(
+                        self.request_interval,
+                        mul_spread=1.1,
+                        max_sleep=self.request_interval + 2,
+                    )
+
+                except Exception as e:
+                    if task.retry < retry_times:
+                        await cid_queue.put(
+                            CidTask(
+                                prev_cid=task.prev_cid,
+                                cid=cid,
+                                retry=task.retry + 1,
+                            )
+                        )
+                        self.logger.info(
+                            "[Fetcher] Re-queued chapter %s for retry #%d: %s",
+                            cid,
+                            task.retry + 1,
+                            e,
+                        )
+                        backoff = self.backoff_factor * (2**task.retry)
+                        await async_sleep_with_random_delay(
+                            base=backoff,
+                            mul_spread=1.2,
+                            max_sleep=backoff + 3,
+                        )
+                    else:
+                        self.logger.warning(
+                            "[Fetcher] Max retries reached for chapter %s: %s",
+                            cid,
+                            e,
+                        )
+
+                finally:
+                    cid_queue.task_done()
+
+        async def parser_worker(
+            cid_queue: asyncio.Queue[CidTask],
+            html_queue: asyncio.Queue[HtmlTask],
+            save_queue: asyncio.Queue[ChapterDict],
+            retry_times: int,
+        ) -> None:
+            while True:
+                task = await html_queue.get()
+                skip_retry = False
+                try:
+                    chap_json: ChapterDict | None = None
+                    if self.is_restricted_page(task.html_list):
+                        self.logger.info(
+                            "[Parser] Skipped restricted page for cid %s", task.cid
+                        )
+                        skip_retry = True
+                    else:
+                        chap_json = await asyncio.to_thread(
+                            self.parser.parse_chapter,
+                            task.html_list,
+                            task.cid,
+                        )
+                    if self.check_encrypted(task.html_list):
+                        skip_retry = True
+                    if chap_json:
+                        await save_queue.put(chap_json)
+                        self.logger.info(
+                            "[Parser] saved chapter %s",
+                            task.cid,
+                        )
+                        if self.save_html:
+                            is_encrypted = chap_json.get("extra", {}).get(
+                                "encrypted", False
+                            )
+                            folder = chapters_html_dir / (
+                                "html_encrypted" if is_encrypted else "html_plain"
+                            )
+                            html_path = folder / f"{task.cid}.html"
+                            save_as_txt(task.html_list[0], html_path, on_exist="skip")
+                            self.logger.debug(
+                                "%s Saved raw HTML for chapter %s to %s",
+                                TAG,
+                                task.cid,
+                                html_path,
+                            )
+                    else:
+                        raise ValueError("Empty parse result")
+                except Exception as e:
+                    if not skip_retry and task.retry < retry_times:
+                        await cid_queue.put(
+                            CidTask(prev_cid=None, cid=task.cid, retry=task.retry + 1)
+                        )
+                        self.logger.info(
+                            "[Parser] Re-queued cid %s for retry #%d: %s",
+                            task.cid,
+                            task.retry + 1,
+                            e,
+                        )
+                    elif not skip_retry:
+                        self.logger.warning(
+                            "[Parser] Max retries reached for cid %s: %s",
+                            task.cid,
+                            e,
+                        )
+                finally:
+                    html_queue.task_done()
+
+        async def storage_worker(
+            normal_cs: ChapterStorage,
+            encrypted_cs: ChapterStorage,
+            save_queue: asyncio.Queue[ChapterDict],
+        ) -> None:
+            nonlocal completed_count
+            while True:
+                item = await save_queue.get()
+                try:
+                    is_encrypted = item.get("extra", {}).get("encrypted", False)
+                    cs = encrypted_cs if is_encrypted else normal_cs
+                    cs.save(cast(ChapterDict, item))
+                    completed_count += 1
+                    if progress_hook:
+                        await progress_hook(completed_count, total_chapters)
+                except Exception as e:
+                    self.logger.error("[storage_worker] Failed to save: %s", e)
+                finally:
+                    save_queue.task_done()
+
+        fetcher_task = asyncio.create_task(
+            fetcher_worker(
+                book_id,
+                cid_queue,
+                html_queue,
+                self.retry_times,
+            )
+        )
+
+        parser_task = asyncio.create_task(
+            parser_worker(
+                cid_queue,
+                html_queue,
+                save_queue,
+                self.retry_times,
+            )
+        )
+
+        storage_task = asyncio.create_task(
+            storage_worker(
+                normal_cs=normal_cs,
+                encrypted_cs=encrypted_cs,
+                save_queue=save_queue,
+            )
+        )
+
+        last_cid: str | None = None
+        for vol in book_info.get("volumes", []):
+            chapters = vol.get("chapters", [])
+            for chap in chapters:
+                cid = chap.get("chapterId")
+                if cid and normal_cs.exists(cid) and self.skip_existing:
+                    completed_count += 1
+                    if progress_hook:
+                        await progress_hook(completed_count, total_chapters)
+                    last_cid = cid
+                    continue
+
+                await cid_queue.put(CidTask(cid=cid, prev_cid=last_cid))
+                last_cid = cid
+
+        await cid_queue.join()
+        await html_queue.join()
+        await save_queue.join()
+
+        for task in [fetcher_task, parser_task, storage_task]:
+            task.cancel()
+            with suppress(asyncio.CancelledError):
+                await task
+
+        normal_cs.close()
+        encrypted_cs.close()
+
+        await asyncio.to_thread(self.exporter.export, book_id)
+
+        self.logger.info(
+            "%s Novel '%s' download completed.",
+            TAG,
+            book_info.get("book_name", "unknown"),
+        )
+        return
+
+    @staticmethod
+    def is_restricted_page(html_list: list[str]) -> bool:
+        """
+        Return True if page content indicates access restriction
+        (e.g. not subscribed/purchased).
+
+        :param html_list: Raw HTML string.
+        """
+        if not html_list:
+            return True
+        markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
+        return any(m in html_list[0] for m in markers)
+
+    @staticmethod
+    def check_encrypted(html_list: list[str]) -> bool:
+        if not html_list:
+            return True
+        return '"cES":2' in html_list[0]

novel_downloader/core/downloaders/sfacg.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.sfacg
+---------------------------------------
+
+"""
+
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ExporterProtocol,
+    FetcherProtocol,
+    ParserProtocol,
+)
+from novel_downloader.models import DownloaderConfig
+
+
+class SfacgDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        fetcher: FetcherProtocol,
+        parser: ParserProtocol,
+        exporter: ExporterProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(fetcher, parser, exporter, config, "sfacg")

novel_downloader/core/downloaders/yamibo.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.yamibo
+----------------------------------------
+
+"""
+
+from novel_downloader.core.downloaders.common import CommonDownloader
+from novel_downloader.core.interfaces import (
+    ExporterProtocol,
+    FetcherProtocol,
+    ParserProtocol,
+)
+from novel_downloader.models import DownloaderConfig
+
+
+class YamiboDownloader(CommonDownloader):
+    """"""
+
+    def __init__(
+        self,
+        fetcher: FetcherProtocol,
+        parser: ParserProtocol,
+        exporter: ExporterProtocol,
+        config: DownloaderConfig,
+    ):
+        super().__init__(fetcher, parser, exporter, config, "yamibo")

novel_downloader/core/exporters/__init__.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.exporters
+-------------------------------
+
+This module defines exporter classes for different novel platforms.
+
+Currently supported platforms:
+- biquge (笔趣阁)
+- esjzone (ESJ Zone)
+- linovelib (哔哩轻小说)
+- qianbi (铅笔小说)
+- qidian (起点中文网)
+- sfacg (SF轻小说)
+- yamibo (百合会)
+- common (通用架构)
+"""
+
+from .biquge import BiqugeExporter
+from .common import CommonExporter
+from .esjzone import EsjzoneExporter
+from .linovelib import LinovelibExporter
+from .qianbi import QianbiExporter
+from .qidian import QidianExporter
+from .sfacg import SfacgExporter
+from .yamibo import YamiboExporter
+
+__all__ = [
+    "BiqugeExporter",
+    "EsjzoneExporter",
+    "LinovelibExporter",
+    "QianbiExporter",
+    "QidianExporter",
+    "SfacgExporter",
+    "YamiboExporter",
+    "CommonExporter",
+]

novel_downloader/core/{savers → exporters}/base.py

@@ -1,21 +1,22 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.savers.base
----------------------------------
+novel_downloader.core.exporters.base
+------------------------------------
 
-This module provides an abstract base class `BaseSaver` that defines the
-common interface and reusable logic for saving assembled novel content
-into various output formats.
+This module provides an abstract base class `BaseExporter` that defines
+the common interface and reusable logic for saving assembled novel
+content into various output formats.
 """
 
 import abc
 import logging
+import types
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Any, Self
 
-from novel_downloader.config.models import SaverConfig
-from novel_downloader.core.interfaces import SaverProtocol
+from novel_downloader.core.interfaces import ExporterProtocol
+from novel_downloader.models import ExporterConfig
 
 
 class SafeDict(dict[str, Any]):
@@ -23,61 +24,67 @@ class SafeDict(dict[str, Any]):
         return f"{{{key}}}"
 
 
-class BaseSaver(SaverProtocol, abc.ABC):
+class BaseExporter(ExporterProtocol, abc.ABC):
     """
-    BaseSaver defines the interface and common structure for
+    BaseExporter defines the interface and common structure for
     saving assembled book content into various formats
     such as TXT, EPUB, Markdown, or PDF.
     """
 
-    def __init__(self, config: SaverConfig):
+    def __init__(
+        self,
+        config: ExporterConfig,
+        site: str,
+    ):
         """
-        Initialize the saver with given configuration.
+        Initialize the exporter with given configuration.
 
-        :param config: A SaverConfig object that defines
+        :param config: A ExporterConfig object that defines
                         save paths, formats, and options.
         """
         self._config = config
+        self._site = site
 
-        self._base_cache_dir = Path(config.cache_dir)
-        self._raw_data_dir = Path(config.raw_data_dir)
+        self._cache_dir = Path(config.cache_dir) / site
+        self._raw_data_dir = Path(config.raw_data_dir) / site
         self._output_dir = Path(config.output_dir)
-        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
+        self._cache_dir.mkdir(parents=True, exist_ok=True)
         self._output_dir.mkdir(parents=True, exist_ok=True)
 
-        self._filename_template = config.filename_template
-
-        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+        self.logger = logging.getLogger(f"{self.__class__.__name__}")
 
-    def save(self, book_id: str) -> None:
+    def export(
+        self,
+        book_id: str,
+    ) -> None:
         """
-        Save the book in the formats specified in config.
+        Export the book in the formats specified in config.
         If a method is not implemented or fails, log the error and continue.
 
         :param book_id: The book identifier (used for filename, lookup, etc.)
         """
-        TAG = "[Saver]"
+        TAG = "[Exporter]"
         actions = [
-            ("make_txt", self.save_as_txt),
-            ("make_epub", self.save_as_epub),
-            ("make_md", self.save_as_md),
-            ("make_pdf", self.save_as_pdf),
+            ("make_txt", self.export_as_txt),
+            ("make_epub", self.export_as_epub),
+            ("make_md", self.export_as_md),
+            ("make_pdf", self.export_as_pdf),
         ]
 
-        for flag_name, save_method in actions:
+        for flag_name, export_method in actions:
             if getattr(self._config, flag_name, False):
                 try:
                     self.logger.info(
-                        "%s Attempting to save book_id '%s' as %s...",
+                        "%s Attempting to export book_id '%s' as %s...",
                         TAG,
                         book_id,
                         flag_name,
                     )
-                    save_method(book_id)
+                    export_method(book_id)
                     self.logger.info("%s Successfully saved as %s.", TAG, flag_name)
                 except NotImplementedError as e:
                     self.logger.warning(
-                        "%s Save method for %s not implemented: %s",
+                        "%s Export method for %s not implemented: %s",
                         TAG,
                         flag_name,
                         str(e),
@@ -89,7 +96,7 @@ class BaseSaver(SaverProtocol, abc.ABC):
         return
 
     @abc.abstractmethod
-    def save_as_txt(self, book_id: str) -> None:
+    def export_as_txt(self, book_id: str) -> None:
         """
         Persist the assembled book as a .txt file.
 
@@ -99,32 +106,32 @@ class BaseSaver(SaverProtocol, abc.ABC):
         """
         ...
 
-    def save_as_epub(self, book_id: str) -> None:
+    def export_as_epub(self, book_id: str) -> None:
         """
         Optional: Persist the assembled book as a EPUB (.epub) file.
 
         :param book_id: The book identifier.
         :raises NotImplementedError: If the method is not overridden.
         """
-        raise NotImplementedError("EPUB export not supported by this saver.")
+        raise NotImplementedError("EPUB export not supported by this Exporter.")
 
-    def save_as_md(self, book_id: str) -> None:
+    def export_as_md(self, book_id: str) -> None:
         """
         Optional: Persist the assembled book as a Markdown file.
 
         :param book_id: The book identifier.
         :raises NotImplementedError: If the method is not overridden.
         """
-        raise NotImplementedError("Markdown export not supported by this saver.")
+        raise NotImplementedError("Markdown export not supported by this Exporter.")
 
-    def save_as_pdf(self, book_id: str) -> None:
+    def export_as_pdf(self, book_id: str) -> None:
         """
         Optional: Persist the assembled book as a PDF file.
 
         :param book_id: The book identifier.
         :raises NotImplementedError: If the method is not overridden.
         """
-        raise NotImplementedError("PDF export not supported by this saver.")
+        raise NotImplementedError("PDF export not supported by this Exporter.")
 
     def get_filename(
         self,
@@ -146,7 +153,7 @@ class BaseSaver(SaverProtocol, abc.ABC):
         # Merge all fields with defaults
         context = SafeDict(title=title, author=author or "", **extra_fields)
 
-        name = self._filename_template.format_map(context)
+        name = self.filename_template.format_map(context)
 
         if self._config.append_timestamp:
             name += f"_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
@@ -158,12 +165,34 @@ class BaseSaver(SaverProtocol, abc.ABC):
         """Access the output directory for saving files."""
         return self._output_dir
 
-    @property
-    def raw_data_dir(self) -> Path:
-        """Access the raw data directory."""
-        return self._raw_data_dir
-
     @property
     def filename_template(self) -> str:
         """Access the filename template."""
-        return self._filename_template
+        return self._config.filename_template
+
+    def _on_close(self) -> None:
+        """
+        Hook method called at the beginning of close().
+        Override in subclass if needed.
+        """
+        pass
+
+    def close(self) -> None:
+        """
+        Shutdown and clean up the exporter.
+        """
+        self._on_close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        tb: types.TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        self.close()

novel_downloader/core/exporters/biquge.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.exporters.biquge
+--------------------------------------
+
+"""
+
+from novel_downloader.models import ExporterConfig
+
+from .common import CommonExporter
+
+
+class BiqugeExporter(CommonExporter):
+    def __init__(
+        self,
+        config: ExporterConfig,
+    ):
+        super().__init__(
+            config,
+            site="biquge",
+            chap_folders=["chapters"],
+        )
+
+
+__all__ = ["BiqugeExporter"]