novel-downloader 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

novel_downloader/core/downloaders/base.py

@@ -10,15 +10,11 @@ import abc
 import asyncio
 import json
 import logging
-from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
+from collections.abc import Awaitable, Callable, Sequence
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, ClassVar, cast
 
-from novel_downloader.core.interfaces import (
-    DownloaderProtocol,
-    FetcherProtocol,
-    ParserProtocol,
-)
+from novel_downloader.core.interfaces import FetcherProtocol, ParserProtocol
 from novel_downloader.models import (
     BookConfig,
     BookInfoDict,
@@ -28,7 +24,7 @@ from novel_downloader.models import (
 from novel_downloader.utils import time_diff
 
 
-class BaseDownloader(DownloaderProtocol, abc.ABC):
+class BaseDownloader(abc.ABC):
     """
     Abstract base class for novel downloaders.
 
@@ -39,8 +35,8 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
     a single book, using the provided fetcher and parser components.
     """
 
-    DEFAULT_SOURCE_ID = 0
-    PRIORITIES_MAP = {
+    DEFAULT_SOURCE_ID: ClassVar[int] = 0
+    PRIORITIES_MAP: ClassVar[dict[int, int]] = {
         DEFAULT_SOURCE_ID: 0,
     }
 
@@ -61,15 +57,23 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         self._fetcher = fetcher
         self._parser = parser
-        self._config = config
         self._site = site
 
+        self._save_html = config.save_html
+        self._skip_existing = config.skip_existing
+        self._login_required = config.login_required
+        self._request_interval = config.request_interval
+        self._retry_times = config.retry_times
+        self._backoff_factor = config.backoff_factor
+        self._workers = config.workers
+        self._storage_batch_size = max(1, config.storage_batch_size)
+
         self._raw_data_dir = Path(config.raw_data_dir) / site
         self._raw_data_dir.mkdir(parents=True, exist_ok=True)
         self._debug_dir = Path.cwd() / "debug" / site
         self._debug_dir.mkdir(parents=True, exist_ok=True)
 
-        self.logger = logging.getLogger(f"{self.__class__.__name__}")
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
     async def download_many(
         self,
@@ -87,7 +91,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                                 args: completed_count, total_count.
         :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
-        if not await self._ensure_ready():
+        if not self._check_login():
             book_ids = [b["book_id"] for b in books]
             self.logger.warning(
                 "[%s] login failed, skipping download of books: %s",
@@ -116,8 +120,6 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
             except Exception as e:
                 self._handle_download_exception(book, e)
 
-        await self._finalize()
-
     async def download(
         self,
         book: BookConfig,
@@ -134,7 +136,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                                 args: completed_count, total_count.
         :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
-        if not await self._ensure_ready():
+        if not self._check_login():
             self.logger.warning(
                 "[%s] login failed, skipping download of book: %s (%s-%s)",
                 self._site,
@@ -142,6 +144,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                 book.get("start_id", "-"),
                 book.get("end_id", "-"),
             )
+            return
 
         # if already cancelled before starting
         if cancel_event and cancel_event.is_set():
@@ -162,8 +165,6 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         except Exception as e:
             self._handle_download_exception(book, e)
 
-        await self._finalize()
-
     async def load_book_info(
         self,
         book_id: str,
@@ -200,23 +201,6 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         ...
 
-    async def _prepare(self) -> None:
-        """
-        Optional hook called before downloading.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    async def _finalize(self) -> None:
-        """
-        Optional hook called after downloading is complete.
-
-        Subclasses can override this method to perform post-download tasks,
-        such as saving state or releasing resources.
-        """
-        return
-
     def _load_book_info(
         self,
         book_id: str,
@@ -283,25 +267,22 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param filename: used as filename prefix
         :param html_list: list of HTML strings to save
         """
-        if not self.save_html:
+        if not self._save_html:
             return
-
         html_dir.mkdir(parents=True, exist_ok=True)
         for i, html in enumerate(html_list):
-            file_path = html_dir / f"{filename}_{i}.html"
-            file_path.write_text(html, encoding="utf-8")
+            (html_dir / f"{filename}_{i}.html").write_text(html, encoding="utf-8")
 
     @staticmethod
-    async def _chapter_ids(
-        volumes: list[VolumeInfoDict],
+    def _planned_chapter_ids(
+        vols: list[VolumeInfoDict],
         start_id: str | None,
         end_id: str | None,
-    ) -> AsyncIterator[str]:
-        """
-        Yield each chapterId in order, respecting start/end bounds.
-        """
+        ignore: set[str],
+    ) -> list[str]:
         seen_start = start_id is None
-        for vol in volumes:
+        out: list[str] = []
+        for vol in vols:
             for chap in vol["chapters"]:
                 cid = chap.get("chapterId")
                 if not cid:
@@ -311,9 +292,11 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                         seen_start = True
                     else:
                         continue
-                yield cid
+                if cid not in ignore:
+                    out.append(cid)
                 if end_id is not None and cid == end_id:
-                    return
+                    return out
+        return out
 
     @property
     def fetcher(self) -> FetcherProtocol:
@@ -323,38 +306,6 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
     def parser(self) -> ParserProtocol:
         return self._parser
 
-    @property
-    def save_html(self) -> bool:
-        return self._config.save_html
-
-    @property
-    def skip_existing(self) -> bool:
-        return self._config.skip_existing
-
-    @property
-    def login_required(self) -> bool:
-        return self._config.login_required
-
-    @property
-    def request_interval(self) -> float:
-        return self._config.request_interval
-
-    @property
-    def retry_times(self) -> int:
-        return self._config.retry_times
-
-    @property
-    def backoff_factor(self) -> float:
-        return self._config.backoff_factor
-
-    @property
-    def workers(self) -> int:
-        return self._config.workers
-
-    @property
-    def storage_batch_size(self) -> int:
-        return max(1, self._config.storage_batch_size)
-
     def _handle_download_exception(self, book: BookConfig, error: Exception) -> None:
         """
         Handle download errors in a consistent way.
@@ -373,10 +324,8 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
             error,
         )
 
-    async def _ensure_ready(self) -> bool:
+    def _check_login(self) -> bool:
         """
-        Run pre-download preparation and check login if needed.
+        Check login if needed.
         """
-        await self._prepare()
-
-        return self.fetcher.is_logged_in if self.login_required else True
+        return self.fetcher.is_logged_in if self._login_required else True

novel_downloader/core/downloaders/common.py

@@ -12,19 +12,9 @@ from pathlib import Path
 from typing import Any
 
 from novel_downloader.core.downloaders.base import BaseDownloader
-from novel_downloader.core.downloaders.signals import (
-    STOP,
-    Progress,
-    StopToken,
-)
-from novel_downloader.models import (
-    BookConfig,
-    ChapterDict,
-)
-from novel_downloader.utils import (
-    ChapterStorage,
-    async_jitter_sleep,
-)
+from novel_downloader.core.downloaders.signals import STOP, Progress, StopToken
+from novel_downloader.models import BookConfig, ChapterDict
+from novel_downloader.utils import ChapterStorage, async_jitter_sleep
 
 
 class CommonDownloader(BaseDownloader):
@@ -41,7 +31,7 @@ class CommonDownloader(BaseDownloader):
         **kwargs: Any,
     ) -> None:
         """
-        Sentinel-based pipeline with graceful cancellation:
+        Sentinel-based pipeline with cancellation:
 
         Producer -> ChapterWorkers -> StorageWorker.
 
@@ -59,181 +49,167 @@ class CommonDownloader(BaseDownloader):
         raw_base.mkdir(parents=True, exist_ok=True)
         html_dir = self._debug_dir / book_id / "html"
 
-        chapter_storage = ChapterStorage(
-            raw_base=raw_base,
-            priorities=self.PRIORITIES_MAP,
-        )
-        chapter_storage.connect()
-
         def cancelled() -> bool:
             return bool(cancel_event and cancel_event.is_set())
 
-        try:
-            # --- metadata ---
-            book_info = await self.load_book_info(book_id=book_id, html_dir=html_dir)
-            if not book_info:
-                return
-
-            vols = book_info["volumes"]
-            total_chapters = sum(len(v["chapters"]) for v in vols)
-            if total_chapters == 0:
-                self.logger.warning("%s 书籍没有章节可下载: %s", TAG, book_id)
-                return
+        # --- metadata ---
+        book_info = await self.load_book_info(book_id=book_id, html_dir=html_dir)
+        if not book_info:
+            return
 
-            progress = Progress(total_chapters, progress_hook)
+        vols = book_info["volumes"]
+        plan = self._planned_chapter_ids(vols, start_id, end_id, ignore_set)
+        if not plan:
+            self.logger.info("%s nothing to do after filtering: %s", TAG, book_id)
+            return
 
-            # --- queues & batching ---
-            cid_q: asyncio.Queue[str | StopToken] = asyncio.Queue()
-            save_q: asyncio.Queue[ChapterDict | StopToken] = asyncio.Queue()
-            batch: list[ChapterDict] = []
+        progress = Progress(total=len(plan), hook=progress_hook)
 
-            async def flush_batch() -> None:
-                if not batch:
-                    return
-                try:
-                    chapter_storage.upsert_chapters(batch, self.DEFAULT_SOURCE_ID)
-                except Exception as e:
-                    self.logger.error(
-                        "[Storage] batch upsert failed (size=%d): %s",
-                        len(batch),
-                        e,
-                        exc_info=True,
-                    )
-                else:
-                    await progress.bump(len(batch))
-                finally:
-                    batch.clear()
-
-            # --- stage: storage worker ---
-            async def storage_worker() -> None:
-                """
-                Consumes parsed chapters, writes in batches.
-
-                Terminates after receiving STOP from each chapter worker.
-
-                On cancel: keeps consuming (to avoid blocking producers),
-                flushes, and exits once all STOPs are seen.
-                """
-                stop_count = 0
-                while True:
-                    item = await save_q.get()
-                    if isinstance(item, StopToken):
-                        stop_count += 1
-                        if stop_count == self.workers:
-                            # All chapter workers have exited.
-                            await flush_batch()
-                            return
-                        # else keep waiting for remaining STOPs
-                        continue
-
-                    # Normal chapter
-                    batch.append(item)
-                    if len(batch) >= self.storage_batch_size:
-                        await flush_batch()
+        # --- queues & batching ---
+        cid_q: asyncio.Queue[str | StopToken] = asyncio.Queue(maxsize=self._workers * 2)
+        save_q: asyncio.Queue[ChapterDict | StopToken] = asyncio.Queue(
+            maxsize=self._workers * 2
+        )
+        batch: list[ChapterDict] = []
 
-                    if cancelled():
-                        # Drain whatever is already in the queue
-                        try:
-                            while True:
-                                nxt = save_q.get_nowait()
-                                if isinstance(nxt, StopToken):
-                                    stop_count += 1
-                                else:
-                                    batch.append(nxt)
-                        except asyncio.QueueEmpty:
-                            pass
-                        # Final flush of everything
+        async def flush_batch() -> None:
+            if not batch:
+                return
+            try:
+                storage.upsert_chapters(batch, self.DEFAULT_SOURCE_ID)
+            except Exception as e:
+                self.logger.error(
+                    "[Storage] batch upsert failed (size=%d): %s",
+                    len(batch),
+                    e,
+                    exc_info=True,
+                )
+            else:
+                await progress.bump(len(batch))
+            finally:
+                batch.clear()
+
+        # --- stage: storage worker ---
+        async def storage_worker() -> None:
+            """
+            Consumes parsed chapters, writes in batches.
+
+            Terminates after receiving STOP from each chapter worker.
+
+            On cancel: keeps consuming (to avoid blocking producers),
+            flushes, and exits once all STOPs are seen.
+            """
+            stop_count = 0
+            while True:
+                item = await save_q.get()
+                if isinstance(item, StopToken):
+                    stop_count += 1
+                    if stop_count == self._workers:
+                        # All chapter workers have exited.
                         await flush_batch()
-                        # Wait for remaining STOPs so chapter workers can finish.
-                        while stop_count < self.workers:
-                            nxt = await save_q.get()
+                        return
+                    # else keep waiting for remaining STOPs
+                    continue
+
+                # Normal chapter
+                batch.append(item)
+                if len(batch) >= self._storage_batch_size:
+                    await flush_batch()
+
+                if cancelled():
+                    # Drain whatever is already in the queue
+                    try:
+                        while True:
+                            nxt = save_q.get_nowait()
                             if isinstance(nxt, StopToken):
                                 stop_count += 1
-                        return
-
-            # --- stage: chapter worker ---
-            sem = asyncio.Semaphore(self.workers)
+                            else:
+                                batch.append(nxt)
+                    except asyncio.QueueEmpty:
+                        pass
+                    # Final flush of everything
+                    await flush_batch()
+                    # Wait for remaining STOPs so chapter workers can finish.
+                    while stop_count < self._workers:
+                        nxt = await save_q.get()
+                        if isinstance(nxt, StopToken):
+                            stop_count += 1
+                    return
 
-            async def chapter_worker() -> None:
-                """
-                Fetch + parse with retry, then enqueue to save_q.
+        # --- stage: chapter worker ---
+        async def chapter_worker() -> None:
+            """
+            Fetch + parse with retry, then enqueue to save_q.
+
+            Exits on STOP, or early if cancel is set before starting a new fetch.
+            """
+            while True:
+                cid = await cid_q.get()
+                if isinstance(cid, StopToken):
+                    # Propagate one STOP to storage and exit.
+                    await save_q.put(STOP)
+                    return
 
-                Exits on STOP, or early if cancel is set before starting a new fetch.
-                """
-                while True:
-                    cid = await cid_q.get()
-                    if isinstance(cid, StopToken):
-                        # Propagate one STOP to storage and exit.
-                        await save_q.put(STOP)
-                        return
+                # If cancelled, don't start a new network call; let storage finish.
+                if cancelled():
+                    await save_q.put(STOP)
+                    return
 
-                    if not cid or cid in ignore_set:
-                        # Ignore silently and continue.
-                        continue
+                chap = await self._process_chapter(book_id, cid, html_dir)
+                if chap:
+                    await save_q.put(chap)
 
-                    # If cancelled, don't start a new network call; let storage finish.
-                    if cancelled():
-                        await save_q.put(STOP)
-                        return
+                # polite pacing
+                await async_jitter_sleep(
+                    self._request_interval,
+                    mul_spread=1.1,
+                    max_sleep=self._request_interval + 2,
+                )
 
-                    async with sem:
-                        chap = await self._process_chapter(book_id, cid, html_dir)
-                    if chap:
-                        await save_q.put(chap)
+        # --- stage: producer ---
+        async def producer() -> None:
+            """
+            Enqueue chapter IDs (respecting start/end/skip_existing).
 
-                    # polite pacing
-                    await async_jitter_sleep(
-                        self.request_interval,
-                        mul_spread=1.1,
-                        max_sleep=self.request_interval + 2,
-                    )
-
-            # --- stage: producer ---
-            async def producer() -> None:
-                """
-                Enqueue chapter IDs (respecting start/end/skip_existing).
-
-                Always sends STOP x workers at the end (even if cancelled early),
-                so chapter workers can exit deterministically.
-                """
-                try:
-                    async for cid in self._chapter_ids(vols, start_id, end_id):
-                        if cancelled():
-                            break
-                        if self.skip_existing and chapter_storage.exists(cid):
-                            # Count as completed but don't enqueue.
-                            await progress.bump(1)
-                        else:
-                            await cid_q.put(cid)
-                finally:
-                    for _ in range(self.workers):
-                        await cid_q.put(STOP)
-
-            # --- run the pipeline ---
+            Always sends STOP x workers at the end (even if cancelled early),
+            so chapter workers can exit deterministically.
+            """
+            try:
+                for cid in plan:
+                    if cancelled():
+                        break
+                    if self._skip_existing and storage.exists(cid):
+                        # Count as completed but don't enqueue.
+                        await progress.bump(1)
+                    else:
+                        await cid_q.put(cid)
+            finally:
+                for _ in range(self._workers):
+                    await cid_q.put(STOP)
+
+        # --- run the pipeline ---
+        with ChapterStorage(raw_base, priorities=self.PRIORITIES_MAP) as storage:
             async with asyncio.TaskGroup() as tg:
                 tg.create_task(storage_worker())
-                for _ in range(self.workers):
+                for _ in range(self._workers):
                     tg.create_task(chapter_worker())
                 tg.create_task(producer())
 
-            # --- done ---
-            if cancelled():
-                self.logger.info(
-                    "%s Novel '%s' cancelled: flushed %d/%d chapters.",
-                    TAG,
-                    book_info.get("book_name", "unknown"),
-                    progress.done,
-                    progress.total,
-                )
-            else:
-                self.logger.info(
-                    "%s Novel '%s' download completed.",
-                    TAG,
-                    book_info.get("book_name", "unknown"),
-                )
-
-        finally:
-            chapter_storage.close()
+        # --- done ---
+        if cancelled():
+            self.logger.info(
+                "%s Novel '%s' cancelled: flushed %d/%d chapters.",
+                TAG,
+                book_info.get("book_name", "unknown"),
+                progress.done,
+                progress.total,
+            )
+        else:
+            self.logger.info(
+                "%s Novel '%s' download completed.",
+                TAG,
+                book_info.get("book_name", "unknown"),
+            )
 
     async def _process_chapter(
         self,
@@ -247,7 +223,7 @@ class CommonDownloader(BaseDownloader):
 
         :return: ChapterDict on success, or None on failure.
         """
-        for attempt in range(self.retry_times + 1):
+        for attempt in range(self._retry_times + 1):
             try:
                 html_list = await self.fetcher.get_book_chapter(book_id, cid)
                 self._save_html_pages(html_dir, cid, html_list)
@@ -258,11 +234,11 @@ class CommonDownloader(BaseDownloader):
                     raise ValueError("Empty parse result")
                 return chap
             except Exception as e:
-                if attempt < self.retry_times:
+                if attempt < self._retry_times:
                     self.logger.info(
                         "[ChapterWorker] Retry %s (%s): %s", cid, attempt + 1, e
                     )
-                    backoff = self.backoff_factor * (2**attempt)
+                    backoff = self._backoff_factor * (2**attempt)
                     await async_jitter_sleep(
                         base=backoff, mul_spread=1.2, max_sleep=backoff + 3
                     )