novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/downloaders/base.py

@@ -3,32 +3,47 @@
 novel_downloader.core.downloaders.base
 --------------------------------------
 
-Defines the abstract base class `BaseDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
+Abstract base class providing common workflow and utilities for novel downloaders
 """
 
 import abc
+import asyncio
+import json
 import logging
-from collections.abc import Awaitable, Callable
+from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from novel_downloader.core.interfaces import (
     DownloaderProtocol,
     FetcherProtocol,
     ParserProtocol,
 )
-from novel_downloader.models import BookConfig, DownloaderConfig
+from novel_downloader.models import (
+    BookConfig,
+    BookInfoDict,
+    DownloaderConfig,
+    VolumeInfoDict,
+)
+from novel_downloader.utils import time_diff
 
 
 class BaseDownloader(DownloaderProtocol, abc.ABC):
     """
-    Abstract downloader that defines the initialization interface
-    and the general batch download flow.
+    Abstract base class for novel downloaders.
+
+    Defines the general interface and batch download workflow,
+    while delegating book-specific downloading logic to subclasses.
 
-    Subclasses must implement the logic for downloading a single book.
+    Subclasses are required to implement methods for downloading
+    a single book, using the provided fetcher and parser components.
     """
 
+    DEFAULT_SOURCE_ID = 0
+    PRIORITIES_MAP = {
+        DEFAULT_SOURCE_ID: 0,
+    }
+
     def __init__(
         self,
         fetcher: FetcherProtocol,
@@ -36,15 +51,23 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         config: DownloaderConfig,
         site: str,
     ):
+        """
+        Initialize the downloader for a specific site.
+
+        :param fetcher: Fetcher component for retrieving raw chapter data.
+        :param parser: Parser component for extracting chapter content.
+        :param config: Downloader configuration settings.
+        :param site: Identifier for the target website or source.
+        """
         self._fetcher = fetcher
         self._parser = parser
         self._config = config
         self._site = site
 
         self._raw_data_dir = Path(config.raw_data_dir) / site
-        self._cache_dir = Path(config.cache_dir) / site
         self._raw_data_dir.mkdir(parents=True, exist_ok=True)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
+        self._debug_dir = Path.cwd() / "debug" / site
+        self._debug_dir.mkdir(parents=True, exist_ok=True)
 
         self.logger = logging.getLogger(f"{self.__class__.__name__}")
 
@@ -53,6 +76,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         books: list[BookConfig],
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -61,6 +85,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param books: List of BookConfig entries.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         if not await self._ensure_ready():
             book_ids = [b["book_id"] for b in books]
@@ -72,10 +97,20 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
             return
 
         for book in books:
+            # stop early if cancellation requested
+            if cancel_event and cancel_event.is_set():
+                self.logger.info(
+                    "[%s] download cancelled before book: %s",
+                    self._site,
+                    book["book_id"],
+                )
+                break
+
             try:
                 await self._download_one(
                     book,
                     progress_hook=progress_hook,
+                    cancel_event=cancel_event,
                     **kwargs,
                 )
             except Exception as e:
@@ -88,6 +123,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         book: BookConfig,
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -96,6 +132,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param book: BookConfig with at least 'book_id'.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         if not await self._ensure_ready():
             self.logger.warning(
@@ -106,10 +143,20 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                 book.get("end_id", "-"),
             )
 
+        # if already cancelled before starting
+        if cancel_event and cancel_event.is_set():
+            self.logger.info(
+                "[%s] download cancelled before start of book: %s",
+                self._site,
+                book["book_id"],
+            )
+            return
+
         try:
             await self._download_one(
                 book,
                 progress_hook=progress_hook,
+                cancel_event=cancel_event,
                 **kwargs,
             )
         except Exception as e:
@@ -117,12 +164,35 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
 
         await self._finalize()
 
+    async def load_book_info(
+        self,
+        book_id: str,
+        html_dir: Path,
+    ) -> BookInfoDict | None:
+        book_info = self._load_book_info(
+            book_id=book_id,
+            max_age_days=1,
+        )
+        if book_info:
+            return book_info
+
+        info_html = await self.fetcher.get_book_info(book_id)
+        self._save_html_pages(html_dir, "info", info_html)
+        book_info = self.parser.parse_book_info(info_html)
+
+        if book_info:
+            self._save_book_info(book_id, book_info)
+            return book_info
+
+        return self._load_book_info(book_id)
+
     @abc.abstractmethod
     async def _download_one(
         self,
         book: BookConfig,
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -147,29 +217,111 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         return
 
-    @property
-    def fetcher(self) -> FetcherProtocol:
-        return self._fetcher
+    def _load_book_info(
+        self,
+        book_id: str,
+        *,
+        max_age_days: int | None = None,
+    ) -> BookInfoDict | None:
+        """
+        Attempt to read and parse the book_info.json for a given book_id.
 
-    @property
-    def parser(self) -> ParserProtocol:
-        return self._parser
+        :param book_id: identifier of the book
+        :param max_age_days: if set, only return if 'update_time' is less
+        :return: dict of book info if is valid JSON, else empty
+        """
+        info_path = self._raw_data_dir / book_id / "book_info.json"
+        if not info_path.is_file():
+            return None
 
-    @property
-    def config(self) -> DownloaderConfig:
-        return self._config
+        try:
+            raw: dict[str, Any] = json.loads(info_path.read_text(encoding="utf-8"))
+        except json.JSONDecodeError:
+            return None
+
+        if max_age_days is not None:
+            days, *_ = time_diff(
+                raw.get("update_time", ""),
+                "UTC+8",
+            )
+            if days > max_age_days:
+                return None
 
-    @property
-    def raw_data_dir(self) -> Path:
-        return self._raw_data_dir
+        # return data
+        return cast(BookInfoDict, raw)
+
+    def _save_book_info(
+        self,
+        book_id: str,
+        book_info: BookInfoDict,
+    ) -> None:
+        """
+        Serialize and save the book_info dict as json.
+
+        :param book_id: identifier of the book
+        :param book_info: dict containing metadata about the book
+        """
+        target_dir = self._raw_data_dir / book_id
+        target_dir.mkdir(parents=True, exist_ok=True)
+        (target_dir / "book_info.json").write_text(
+            json.dumps(book_info, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+
+    def _save_html_pages(
+        self,
+        html_dir: Path,
+        filename: str,
+        html_list: Sequence[str],
+    ) -> None:
+        """
+        If save_html is enabled, write each HTML snippet to a file.
+
+        Filenames will be {chap_id}_{index}.html in html_dir.
+
+        :param html_dir: directory in which to write HTML files
+        :param filename: used as filename prefix
+        :param html_list: list of HTML strings to save
+        """
+        if not self.save_html:
+            return
+
+        html_dir.mkdir(parents=True, exist_ok=True)
+        for i, html in enumerate(html_list):
+            file_path = html_dir / f"{filename}_{i}.html"
+            file_path.write_text(html, encoding="utf-8")
+
+    @staticmethod
+    async def _chapter_ids(
+        volumes: list[VolumeInfoDict],
+        start_id: str | None,
+        end_id: str | None,
+    ) -> AsyncIterator[str]:
+        """
+        Yield each chapterId in order, respecting start/end bounds.
+        """
+        seen_start = start_id is None
+        for vol in volumes:
+            for chap in vol["chapters"]:
+                cid = chap.get("chapterId")
+                if not cid:
+                    continue
+                if not seen_start:
+                    if cid == start_id:
+                        seen_start = True
+                    else:
+                        continue
+                yield cid
+                if end_id is not None and cid == end_id:
+                    return
 
     @property
-    def cache_dir(self) -> Path:
-        return self._cache_dir
+    def fetcher(self) -> FetcherProtocol:
+        return self._fetcher
 
     @property
-    def site(self) -> str:
-        return self._site
+    def parser(self) -> ParserProtocol:
+        return self._parser
 
     @property
     def save_html(self) -> bool:
@@ -196,12 +348,12 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         return self._config.backoff_factor
 
     @property
-    def parser_workers(self) -> int:
-        return self._config.parser_workers
+    def workers(self) -> int:
+        return self._config.workers
 
     @property
-    def download_workers(self) -> int:
-        return self._config.download_workers
+    def storage_batch_size(self) -> int:
+        return max(1, self._config.storage_batch_size)
 
     def _handle_download_exception(self, book: BookConfig, error: Exception) -> None:
         """