novel-downloader 2.0.1__py3-none-any.whl → 2.0.2__py3-none-any.whl

novel_downloader/core/downloaders/qqbook.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.qqbook
+----------------------------------------
+
+Downloader implementation for QQ novels, with unpurchased chapter ID skip logic.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from pathlib import Path
+from typing import Any, ClassVar
+
+from novel_downloader.core.downloaders.base import BaseDownloader
+from novel_downloader.core.downloaders.registry import register_downloader
+from novel_downloader.core.downloaders.signals import STOP, Progress, StopToken
+from novel_downloader.models import (
+    BookConfig,
+    ChapterDict,
+    VolumeInfoDict,
+)
+from novel_downloader.utils import ChapterStorage, async_jitter_sleep
+
+
+@register_downloader(site_keys=["qqbook", "qq"])
+class QqbookDownloader(BaseDownloader):
+    """
+    Specialized downloader for QQ 阅读 novels.
+
+    Processes each chapter in a single worker that skip non-accessible
+    and handles fetch -> parse -> enqueue storage.
+    """
+
+    DEFAULT_SOURCE_ID: ClassVar[int] = 0
+    ENCRYPTED_SOURCE_ID: ClassVar[int] = 1
+    PRIORITIES_MAP: ClassVar[dict[int, int]] = {
+        DEFAULT_SOURCE_ID: 0,
+        ENCRYPTED_SOURCE_ID: 1,
+    }
+
+    async def _download_one(
+        self,
+        book: BookConfig,
+        *,
+        progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
+        **kwargs: Any,
+    ) -> None:
+        """
+        The full download logic for a single book.
+
+        :param book: BookConfig with at least 'book_id'.
+        """
+        TAG = "[Downloader]"
+        NUM_WORKERS = 1
+
+        book_id = book["book_id"]
+        start_id = book.get("start_id")
+        end_id = book.get("end_id")
+        ignore_set = set(book.get("ignore_ids", []))
+
+        raw_base = self._raw_data_dir / book_id
+        raw_base.mkdir(parents=True, exist_ok=True)
+        html_dir = self._debug_dir / book_id / "html"
+
+        def cancelled() -> bool:
+            return bool(cancel_event and cancel_event.is_set())
+
+        # ---- metadata ---
+        book_info = await self.load_book_info(book_id=book_id, html_dir=html_dir)
+        if not book_info:
+            return
+
+        vols = book_info["volumes"]
+        plan = self._planned_chapter_ids(vols, start_id, end_id, ignore_set)
+        if not plan:
+            self.logger.info("%s nothing to do after filtering: %s", TAG, book_id)
+            return
+
+        progress = Progress(total=len(plan), hook=progress_hook)
+
+        # ---- queues & batching ---
+        cid_q: asyncio.Queue[str | StopToken] = asyncio.Queue(maxsize=self._workers * 2)
+        save_q: asyncio.Queue[ChapterDict | StopToken] = asyncio.Queue(
+            maxsize=self._workers * 2
+        )
+        default_batch: list[ChapterDict] = []
+        encrypted_batch: list[ChapterDict] = []
+
+        def select_batch(chap: ChapterDict) -> tuple[list[ChapterDict], int]:
+            # set extra.encrypted (by parser); default to plain if absent.
+            if chap.get("extra", {}).get("font_encrypt", False):
+                return encrypted_batch, self.ENCRYPTED_SOURCE_ID
+            return default_batch, self.DEFAULT_SOURCE_ID
+
+        async def flush_batch(batch: list[ChapterDict], src: int) -> None:
+            if not batch:
+                return
+            try:
+                storage.upsert_chapters(batch, src)
+            except Exception as e:
+                self.logger.error(
+                    "[Storage] batch upsert failed (size=%d, src=%d): %s",
+                    len(batch),
+                    src,
+                    e,
+                    exc_info=True,
+                )
+            else:
+                await progress.bump(len(batch))
+            finally:
+                batch.clear()
+
+        async def flush_all() -> None:
+            await flush_batch(default_batch, self.DEFAULT_SOURCE_ID)
+            await flush_batch(encrypted_batch, self.ENCRYPTED_SOURCE_ID)
+
+        # ---- workers ---
+        async def storage_worker() -> None:
+            """
+            Consumes parsed chapters, batches by source, flushes on threshold.
+
+            Terminates after receiving STOP from each chapter worker.
+
+            On cancel: drains queue, flushes once, then waits for remaining STOPs.
+            """
+            stop_count = 0
+            while True:
+                chap = await save_q.get()
+                if isinstance(chap, StopToken):
+                    stop_count += 1
+                    if stop_count == NUM_WORKERS:
+                        await flush_all()
+                        return
+                    continue
+
+                batch, src = select_batch(chap)
+                batch.append(chap)
+                if len(batch) >= self._storage_batch_size:
+                    await flush_batch(batch, src)
+
+                if cancelled():
+                    # Drain whatever is already parsed
+                    try:
+                        while True:
+                            nxt = save_q.get_nowait()
+                            if isinstance(nxt, StopToken):
+                                stop_count += 1
+                            else:
+                                nbatch, nsrc = select_batch(nxt)
+                                nbatch.append(nxt)
+                    except asyncio.QueueEmpty:
+                        pass
+                    await flush_all()
+                    # Wait for remaining STOPs to arrive
+                    while stop_count < NUM_WORKERS:
+                        nxt = await save_q.get()
+                        if nxt is STOP:
+                            stop_count += 1
+                    return
+
+        async def chapter_worker() -> None:
+            """
+            Single worker: fetch + parse with retry, then enqueue ChapterDict.
+
+            Exits on STOP. If cancelled, does not start a new fetch; signals STOP.
+            """
+            while True:
+                cid = await cid_q.get()
+                if isinstance(cid, StopToken):
+                    await save_q.put(STOP)
+                    return
+
+                if cancelled():
+                    await save_q.put(STOP)
+                    return
+
+                chap = await self._process_chapter(book_id, cid, html_dir)
+                if chap and not cancelled():
+                    await save_q.put(chap)
+
+                await async_jitter_sleep(
+                    self._request_interval,
+                    mul_spread=1.1,
+                    max_sleep=self._request_interval + 2,
+                )
+
+        async def producer() -> None:
+            """
+            Enqueue chapter IDs respecting start/end/skip_existing.
+
+            Always emits STOP x NUM_WORKERS at the end (even if cancelled early).
+            """
+            try:
+                for cid in plan:
+                    if cancelled():
+                        break
+                    if self._skip_existing and storage.exists(
+                        cid, self.DEFAULT_SOURCE_ID
+                    ):
+                        # Already have not-encrypted; count as done.
+                        await progress.bump(1)
+                    else:
+                        await cid_q.put(cid)
+            finally:
+                for _ in range(NUM_WORKERS):
+                    await cid_q.put(STOP)
+
+        # ---- run tasks ---
+        with ChapterStorage(raw_base, priorities=self.PRIORITIES_MAP) as storage:
+            async with asyncio.TaskGroup() as tg:
+                tg.create_task(storage_worker())
+                for _ in range(NUM_WORKERS):
+                    tg.create_task(chapter_worker())
+                tg.create_task(producer())
+
+        # ---- done ---
+        if cancelled():
+            self.logger.info(
+                "%s Novel '%s' cancelled: flushed %d/%d chapters.",
+                TAG,
+                book_info.get("book_name", "unknown"),
+                progress.done,
+                progress.total,
+            )
+        else:
+            self.logger.info(
+                "%s Novel '%s' download completed.",
+                TAG,
+                book_info.get("book_name", "unknown"),
+            )
+
+    @staticmethod
+    def _planned_chapter_ids(
+        vols: list[VolumeInfoDict],
+        start_id: str | None,
+        end_id: str | None,
+        ignore: set[str],
+    ) -> list[str]:
+        seen_start = start_id is None
+        out: list[str] = []
+        for vol in vols:
+            for chap in vol["chapters"]:
+                cid = chap.get("chapterId")
+                if not cid:
+                    continue
+                if not seen_start:
+                    if cid == start_id:
+                        seen_start = True
+                    else:
+                        continue
+                if cid not in ignore and chap.get("accessible", True):
+                    out.append(cid)
+                if end_id is not None and cid == end_id:
+                    return out
+        return out
+
+    async def _process_chapter(
+        self,
+        book_id: str,
+        cid: str,
+        html_dir: Path,
+    ) -> ChapterDict | None:
+        """
+        Fetch, debug-save, parse a single chapter with retries.
+
+        :return: ChapterDict on success, or None on failure.
+        """
+        for attempt in range(self._retry_times + 1):
+            try:
+                html_list = await self.fetcher.get_book_chapter(book_id, cid)
+                self._save_html_pages(html_dir, cid, html_list)
+                chap = await asyncio.to_thread(
+                    self.parser.parse_chapter, html_list, cid
+                )
+                if not chap:
+                    raise ValueError("Empty parse result")
+                return chap
+            except Exception as e:
+                if attempt < self._retry_times:
+                    self.logger.info(
+                        "[ChapterWorker] Retry %s (%s): %s", cid, attempt + 1, e
+                    )
+                    backoff = self._backoff_factor * (2**attempt)
+                    await async_jitter_sleep(
+                        base=backoff,
+                        mul_spread=1.2,
+                        max_sleep=backoff + 3,
+                    )
+                else:
+                    self.logger.warning("[ChapterWorker] Failed %s: %s", cid, e)
+        return None

novel_downloader/core/downloaders/registry.py

@@ -20,7 +20,7 @@ from novel_downloader.core.interfaces import (
 from novel_downloader.models import DownloaderConfig
 
 DownloaderBuilder = Callable[
-    [FetcherProtocol, ParserProtocol, DownloaderConfig],
+    [FetcherProtocol, ParserProtocol, DownloaderConfig, str],
     DownloaderProtocol,
 ]
 D = TypeVar("D", bound=DownloaderProtocol)
@@ -66,4 +66,4 @@ def get_downloader(
         downloader_cls = _DOWNLOADER_MAP[site_key]
     except KeyError:
         return CommonDownloader(fetcher, parser, config, site_key)
-    return downloader_cls(fetcher, parser, config)
+    return downloader_cls(fetcher, parser, config, site_key)

novel_downloader/core/exporters/__init__.py

@@ -11,9 +11,11 @@ __all__ = [
     "CommonExporter",
     "LinovelibExporter",
     "QidianExporter",
+    "QqbookExporter",
 ]
 
 from .common import CommonExporter
 from .linovelib import LinovelibExporter
 from .qidian import QidianExporter
+from .qqbook import QqbookExporter
 from .registry import get_exporter

novel_downloader/core/exporters/base.py

@@ -7,16 +7,16 @@ Abstract base class providing common structure and utilities for book exporters
 """
 
 import abc
+import contextlib
 import json
 import logging
 import types
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Self, cast
+from typing import Any, ClassVar, Self, cast
 
-from novel_downloader.core.interfaces import ExporterProtocol
 from novel_downloader.models import BookInfoDict, ChapterDict, ExporterConfig
-from novel_downloader.utils import ChapterStorage
+from novel_downloader.utils import ChapterStorage, get_cleaner
 
 
 class SafeDict(dict[str, Any]):
@@ -24,15 +24,15 @@ class SafeDict(dict[str, Any]):
         return f"{{{key}}}"
 
 
-class BaseExporter(ExporterProtocol, abc.ABC):
+class BaseExporter(abc.ABC):
     """
     BaseExporter defines the interface and common structure for
     saving assembled book content into various formats
     such as TXT, EPUB, Markdown, or PDF.
     """
 
-    DEFAULT_SOURCE_ID = 0
-    PRIORITIES_MAP = {
+    DEFAULT_SOURCE_ID: ClassVar[int] = 0
+    PRIORITIES_MAP: ClassVar[dict[int, int]] = {
         DEFAULT_SOURCE_ID: 0,
     }
 
@@ -47,15 +47,30 @@ class BaseExporter(ExporterProtocol, abc.ABC):
         :param config: Exporter configuration settings.
         :param site: Identifier for the target website or source.
         """
-        self._config = config
         self._site = site
         self._storage_cache: dict[str, ChapterStorage] = {}
 
+        self._make_txt = config.make_txt
+        self._make_epub = config.make_epub
+        self._make_md = config.make_md
+        self._make_pdf = config.make_pdf
+
+        self._include_cover = config.include_cover
+        self._include_picture = config.include_picture
+        self._split_mode = config.split_mode
+        self._filename_template = config.filename_template
+        self._append_timestamp = config.append_timestamp
+
         self._raw_data_dir = Path(config.raw_data_dir) / site
         self._output_dir = Path(config.output_dir)
         self._output_dir.mkdir(parents=True, exist_ok=True)
 
-        self.logger = logging.getLogger(f"{self.__class__.__name__}")
+        self._cleaner = get_cleaner(
+            enabled=config.clean_text,
+            config=config.cleaner_cfg,
+        )
+
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
     def export(self, book_id: str) -> dict[str, Path]:
         """
@@ -67,14 +82,14 @@ class BaseExporter(ExporterProtocol, abc.ABC):
         results: dict[str, Path] = {}
 
         actions = [
-            ("make_txt", "txt", self.export_as_txt),
-            ("make_epub", "epub", self.export_as_epub),
-            ("make_md", "md", self.export_as_md),
-            ("make_pdf", "pdf", self.export_as_pdf),
+            (self._make_txt, "txt", self.export_as_txt),
+            (self._make_epub, "epub", self.export_as_epub),
+            (self._make_md, "md", self.export_as_md),
+            (self._make_pdf, "pdf", self.export_as_pdf),
         ]
 
-        for flag_name, fmt_key, export_method in actions:
-            if getattr(self._config, flag_name, False):
+        for enabled, fmt_key, export_method in actions:
+            if enabled:
                 try:
                     self.logger.info(
                         "%s Attempting to export book_id '%s' as %s...",
@@ -93,16 +108,13 @@ class BaseExporter(ExporterProtocol, abc.ABC):
                         "%s Export method for %s not implemented: %s",
                         TAG,
                         fmt_key,
-                        str(e),
-                    )
-                except Exception as e:
-                    self.logger.error(
-                        "%s Error while saving as %s: %s", TAG, fmt_key, str(e)
+                        e,
                     )
+                except Exception:
+                    self.logger.exception("%s Error while saving as %s", TAG, fmt_key)
 
         return results
 
-    @abc.abstractmethod
     def export_as_txt(self, book_id: str) -> Path | None:
         """
         Persist the assembled book as a .txt file.
@@ -111,7 +123,7 @@ class BaseExporter(ExporterProtocol, abc.ABC):
 
         :param book_id: The book identifier (used for filename, lookup, etc.)
         """
-        ...
+        raise NotImplementedError("TXT export not supported by this Exporter.")
 
     def export_as_epub(self, book_id: str) -> Path | None:
         """
@@ -157,39 +169,12 @@ class BaseExporter(ExporterProtocol, abc.ABC):
         :param extra_fields: Any additional fields used in the filename template.
         :return: Formatted filename with extension.
         """
-        # Merge all fields with defaults
         context = SafeDict(title=title, author=author or "", **extra_fields)
-
-        name = self.filename_template.format_map(context)
-
-        if self._config.append_timestamp:
+        name = self._filename_template.format_map(context)
+        if self._append_timestamp:
             name += f"_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-
         return f"{name}.{ext}"
 
-    @property
-    def site(self) -> str:
-        """
-        Get the site identifier.
-
-        :return: The site string.
-        """
-        return self._site
-
-    @property
-    def output_dir(self) -> Path:
-        """
-        Access the output directory for saving files.
-        """
-        return self._output_dir
-
-    @property
-    def filename_template(self) -> str:
-        """
-        Access the filename template.
-        """
-        return self._config.filename_template
-
     def _get_chapter(
         self,
         book_id: str,
@@ -245,18 +230,10 @@ class BaseExporter(ExporterProtocol, abc.ABC):
                 self.logger.warning("Failed to close storage %s: %s", storage, e)
         self._storage_cache.clear()
 
-    def _on_close(self) -> None:
-        """
-        Hook method called at the beginning of close().
-        Override in subclass if needed.
-        """
-        pass
-
     def close(self) -> None:
         """
         Shutdown and clean up the exporter.
         """
-        self._on_close()
         self._close_chapter_storages()
 
     def __enter__(self) -> Self:
@@ -271,4 +248,5 @@ class BaseExporter(ExporterProtocol, abc.ABC):
         self.close()
 
     def __del__(self) -> None:
-        self.close()
+        with contextlib.suppress(Exception):
+            self.close()