novel-downloader 1.5.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

novel_downloader/core/archived/xshbook/searcher.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.xshbook.searcher
+-----------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["xshbook"],
+# )
+class XshbookSearcher(BaseSearcher):
+    site_name = "xshbook"
+    priority = 30
+    BASE_URL = "https://www.xshbook.com"
+    SEARCH_URL = "https://www.sososhu.com/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {
+            "q": keyword,
+            "site": "xshbook",
+        }
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath(
+            "//div[contains(@class,'so_list')]//div[contains(@class,'hot')]//div[contains(@class,'item')]"
+        )
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+            a_nodes = row.xpath(".//dl/dt/a[1]")
+            a = a_nodes[0] if a_nodes else None
+            href = a.get("href") if a is not None else ""
+            book_url = cls._abs_url(href)
+            book_id = cls._book_id_from_url(book_url) if book_url else ""
+            if not book_id:
+                continue
+
+            title = (a.text_content() if a is not None else "").strip()
+            author = cls._first_str(row.xpath(".//dl/dt/span[1]/text()"))
+            cover_url = cls._first_str(
+                row.xpath(".//div[contains(@class,'image')]//img/@src")
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results
+
+    @staticmethod
+    def _book_id_from_url(url: str) -> str:
+        tail = url.split("xshbook.com", 1)[-1]
+        tail = tail.strip("/")
+        return tail.replace("/", "-")

novel_downloader/core/downloaders/__init__.py

@@ -3,38 +3,17 @@
 novel_downloader.core.downloaders
 ---------------------------------
 
-This subpackage contains concrete downloader implementations for
-specific novel platforms.
-
-Each downloader is responsible for orchestrating the full lifecycle
-of retrieving, parsing, and saving novel content for a given source.
-
-Currently supported platforms:
-- biquge (笔趣阁)
-- esjzone (ESJ Zone)
-- linovelib (哔哩轻小说)
-- qianbi (铅笔小说)
-- qidian (起点中文网)
-- sfacg (SF轻小说)
-- yamibo (百合会)
+Downloader implementations for retrieving novels from different sources
 """
 
 __all__ = [
     "get_downloader",
-    "BiqugeDownloader",
-    "EsjzoneDownloader",
-    "LinovelibDownloader",
+    "CommonDownloader",
     "QianbiDownloader",
     "QidianDownloader",
-    "SfacgDownloader",
-    "YamiboDownloader",
 ]
 
-from .biquge import BiqugeDownloader
-from .esjzone import EsjzoneDownloader
-from .linovelib import LinovelibDownloader
+from .common import CommonDownloader
 from .qianbi import QianbiDownloader
 from .qidian import QidianDownloader
 from .registry import get_downloader
-from .sfacg import SfacgDownloader
-from .yamibo import YamiboDownloader

novel_downloader/core/downloaders/base.py

@@ -3,24 +3,29 @@
 novel_downloader.core.downloaders.base
 --------------------------------------
 
-Defines the abstract base class `BaseDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
+Abstract base class providing common workflow and utilities for novel downloaders
 """
 
 import abc
+import asyncio
 import json
 import logging
 from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from novel_downloader.core.interfaces import (
     DownloaderProtocol,
     FetcherProtocol,
     ParserProtocol,
 )
-from novel_downloader.models import BookConfig, DownloaderConfig
-from novel_downloader.utils import calculate_time_difference
+from novel_downloader.models import (
+    BookConfig,
+    BookInfoDict,
+    DownloaderConfig,
+    VolumeInfoDict,
+)
+from novel_downloader.utils import time_diff
 
 
 class BaseDownloader(DownloaderProtocol, abc.ABC):
@@ -35,7 +40,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
     """
 
     DEFAULT_SOURCE_ID = 0
-    DEFAULT_PRIORITIES_MAP = {
+    PRIORITIES_MAP = {
         DEFAULT_SOURCE_ID: 0,
     }
 
@@ -45,7 +50,6 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         parser: ParserProtocol,
         config: DownloaderConfig,
         site: str,
-        priorities: dict[int, int] | None = None,
     ):
         """
         Initialize the downloader for a specific site.
@@ -54,15 +58,11 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param parser: Parser component for extracting chapter content.
         :param config: Downloader configuration settings.
         :param site: Identifier for the target website or source.
-        :param priorities: Mapping of source_id to priority value.
-                           Lower numbers indicate higher priority.
-                           E.X. {0: 10, 1: 100} means source 0 is preferred.
         """
         self._fetcher = fetcher
         self._parser = parser
         self._config = config
         self._site = site
-        self._priorities = priorities or self.DEFAULT_PRIORITIES_MAP
 
         self._raw_data_dir = Path(config.raw_data_dir) / site
         self._raw_data_dir.mkdir(parents=True, exist_ok=True)
@@ -76,6 +76,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         books: list[BookConfig],
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -84,6 +85,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param books: List of BookConfig entries.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         if not await self._ensure_ready():
             book_ids = [b["book_id"] for b in books]
@@ -95,10 +97,20 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
             return
 
         for book in books:
+            # stop early if cancellation requested
+            if cancel_event and cancel_event.is_set():
+                self.logger.info(
+                    "[%s] download cancelled before book: %s",
+                    self._site,
+                    book["book_id"],
+                )
+                break
+
             try:
                 await self._download_one(
                     book,
                     progress_hook=progress_hook,
+                    cancel_event=cancel_event,
                     **kwargs,
                 )
             except Exception as e:
@@ -111,6 +123,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         book: BookConfig,
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -119,6 +132,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         :param book: BookConfig with at least 'book_id'.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         if not await self._ensure_ready():
             self.logger.warning(
@@ -129,10 +143,20 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
                 book.get("end_id", "-"),
             )
 
+        # if already cancelled before starting
+        if cancel_event and cancel_event.is_set():
+            self.logger.info(
+                "[%s] download cancelled before start of book: %s",
+                self._site,
+                book["book_id"],
+            )
+            return
+
         try:
             await self._download_one(
                 book,
                 progress_hook=progress_hook,
+                cancel_event=cancel_event,
                 **kwargs,
             )
         except Exception as e:
@@ -144,7 +168,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         self,
         book_id: str,
         html_dir: Path,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         book_info = self._load_book_info(
             book_id=book_id,
             max_age_days=1,
@@ -168,6 +192,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         book: BookConfig,
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -197,7 +222,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         book_id: str,
         *,
         max_age_days: int | None = None,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Attempt to read and parse the book_info.json for a given book_id.
 
@@ -207,27 +232,28 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         info_path = self._raw_data_dir / book_id / "book_info.json"
         if not info_path.is_file():
-            return {}
+            return None
 
         try:
-            data: dict[str, Any] = json.loads(info_path.read_text(encoding="utf-8"))
+            raw: dict[str, Any] = json.loads(info_path.read_text(encoding="utf-8"))
         except json.JSONDecodeError:
-            return {}
+            return None
 
         if max_age_days is not None:
-            days, *_ = calculate_time_difference(
-                data.get("update_time", ""),
+            days, *_ = time_diff(
+                raw.get("update_time", ""),
                 "UTC+8",
             )
             if days > max_age_days:
-                return {}
+                return None
 
-        return data
+        # return data
+        return cast(BookInfoDict, raw)
 
     def _save_book_info(
         self,
         book_id: str,
-        book_info: dict[str, Any],
+        book_info: BookInfoDict,
     ) -> None:
         """
         Serialize and save the book_info dict as json.
@@ -267,7 +293,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
 
     @staticmethod
     async def _chapter_ids(
-        volumes: list[dict[str, Any]],
+        volumes: list[VolumeInfoDict],
         start_id: str | None,
         end_id: str | None,
     ) -> AsyncIterator[str]:
@@ -276,7 +302,7 @@ class BaseDownloader(DownloaderProtocol, abc.ABC):
         """
         seen_start = start_id is None
         for vol in volumes:
-            for chap in vol.get("chapters", []):
+            for chap in vol["chapters"]:
                 cid = chap.get("chapterId")
                 if not cid:
                     continue