novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/searchers/base.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.base
+------------------------------------
+
+Abstract base class providing common utilities for site-specific searchers.
+"""
+
+import abc
+from typing import Any, ClassVar
+from urllib.parse import quote_plus, urljoin
+
+import aiohttp
+
+from novel_downloader.core.interfaces import SearcherProtocol
+from novel_downloader.models import SearchResult
+from novel_downloader.utils.constants import DEFAULT_USER_HEADERS
+
+
+class BaseSearcher(abc.ABC, SearcherProtocol):
+    site_name: str
+    BASE_URL: str = ""
+    _session: ClassVar[aiohttp.ClientSession | None] = None
+
+    @classmethod
+    def configure(cls, session: aiohttp.ClientSession) -> None:
+        cls._session = session
+
+    @classmethod
+    async def search(cls, keyword: str, limit: int | None = None) -> list[SearchResult]:
+        html = await cls._fetch_html(keyword)
+        return cls._parse_html(html, limit)
+
+    @classmethod
+    @abc.abstractmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        """
+        Fetch raw HTML from search API or page
+
+        :param keyword: The search term to query.
+        :return: HTML text of the search results page, or an empty string on fail.
+        """
+        pass
+
+    @classmethod
+    @abc.abstractmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        """
+        Parse raw HTML from search API or page into list of SearchResult.
+
+        :param html_str: Raw HTML string from search results page.
+        :param limit: Maximum number of results to return, or None for all.
+        :return: List of SearchResult dicts.
+        """
+        pass
+
+    @classmethod
+    async def _http_get(
+        cls,
+        url: str,
+        *,
+        params: dict[str, str] | None = None,
+        headers: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> aiohttp.ClientResponse:
+        """
+        Helper for GET requests with default headers.
+        """
+        session = cls._ensure_session()
+        hdrs = {**DEFAULT_USER_HEADERS, **(headers or {})}
+        resp = await session.get(url, params=params, headers=hdrs, **kwargs)
+        try:
+            resp.raise_for_status()
+        except aiohttp.ClientResponseError:
+            try:
+                await resp.read()
+            finally:
+                resp.release()
+            raise
+        return resp
+
+    @classmethod
+    async def _http_post(
+        cls,
+        url: str,
+        *,
+        data: dict[str, str] | str | None = None,
+        headers: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> aiohttp.ClientResponse:
+        """
+        Helper for POST requests with default headers.
+        """
+        session = cls._ensure_session()
+        hdrs = {**DEFAULT_USER_HEADERS, **(headers or {})}
+        resp = await session.post(url, data=data, headers=hdrs, **kwargs)
+        try:
+            resp.raise_for_status()
+        except aiohttp.ClientResponseError:
+            try:
+                await resp.read()
+            finally:
+                resp.release()
+            raise
+        return resp
+
+    @classmethod
+    def _ensure_session(cls) -> aiohttp.ClientSession:
+        if cls._session is None:
+            raise RuntimeError(
+                f"{cls.__name__} has no aiohttp session. "
+                "Call .configure(session) first."
+            )
+        return cls._session
+
+    @staticmethod
+    def _quote(q: str, encoding: str | None = None, errors: str | None = None) -> str:
+        """URL-encode a query string safely."""
+        return quote_plus(q, encoding=encoding, errors=errors)
+
+    @staticmethod
+    async def _response_to_str(
+        resp: aiohttp.ClientResponse,
+        encoding: str | None = None,
+    ) -> str:
+        """
+        Read the full body of resp as text. First try the declared charset,
+        then on UnicodeDecodeError fall back to a lenient utf-8 decode.
+        """
+        data: bytes = await resp.read()
+        encodings = [
+            encoding,
+            resp.charset,
+            "gb2312",
+            "gb18030",
+            "gbk",
+            "utf-8",
+        ]
+        encodings_list: list[str] = [e for e in encodings if e]
+        for enc in encodings_list:
+            try:
+                return data.decode(enc)
+            except UnicodeDecodeError:
+                continue
+        encoding = encoding or "utf-8"
+        return data.decode(encoding, errors="ignore")
+
+    @staticmethod
+    def _first_str(xs: list[str], replaces: list[tuple[str, str]] | None = None) -> str:
+        replaces = replaces or []
+        value: str = xs[0].strip() if xs else ""
+        for replace in replaces:
+            old, new = replace
+            value = value.replace(old, new)
+        return value
+
+    @staticmethod
+    def _build_url(base: str, params: dict[str, str]) -> str:
+        query_string = "&".join(f"{k}={v}" for k, v in params.items())
+        return f"{base}?{query_string}"
+
+    @classmethod
+    def _abs_url(cls, url: str) -> str:
+        return (
+            url
+            if url.startswith(("http://", "https://"))
+            else urljoin(cls.BASE_URL, url)
+        )

novel_downloader/core/searchers/dxmwx.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.dxmwx
+-------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["dxmwx"],
+)
+class DxmwxSearcher(BaseSearcher):
+    site_name = "dxmwx"
+    priority = 30
+    BASE_URL = "https://www.dxmwx.org"
+    SEARCH_URL = "https://www.dxmwx.org/list/{query}.html"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        url = cls.SEARCH_URL.format(query=cls._quote(keyword))
+        try:
+            async with (await cls._http_get(url)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath(
+            "//div[@id='ListContents']/div[contains(@style,'position: relative')]"
+        )
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(
+                row.xpath(".//div[contains(@class,'margin0h5')]//a[1]/@href")
+            )
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            book_url = cls._abs_url(href)
+            # "/book/10409.html" -> "10409"
+            book_id = href.split("/")[-1].split(".", 1)[0]
+
+            title = cls._first_str(
+                row.xpath(".//div[contains(@class,'margin0h5')]//a[1]/text()")
+            )
+
+            author = cls._first_str(
+                row.xpath(".//div[contains(@class,'margin0h5')]//a[2]/text()")
+            )
+
+            cover_src = cls._first_str(
+                row.xpath(".//div[contains(@class,'imgwidth')]//img/@src")
+            )
+            cover_url = cls._abs_url(cover_src) if cover_src else ""
+
+            latest_chapter = cls._first_str(
+                row.xpath(
+                    ".//a[span and span[contains(normalize-space(.),'最新章节')]]"
+                    "/span/following-sibling::text()[1]"
+                )
+            )
+
+            update_date = cls._first_str(
+                row.xpath(".//span[contains(@class,'lefth5')]/text()")
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/eightnovel.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.eightnovel
+------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["eightnovel", "8novel"],
+)
+class EightnovelSearcher(BaseSearcher):
+    site_name = "8novel"
+    priority = 20
+    BASE_URL = "https://www.8novel.com"
+    SEARCH_URL = "https://www.8novel.com/search/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"key": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        anchors = doc.xpath("//div[contains(@class,'picsize')]/a")
+        results: list[SearchResult] = []
+
+        for idx, a in enumerate(anchors):
+            href = cls._first_str(a.xpath("./@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # '/novelbooks/6045' -> "6045"
+            book_id = href.rstrip("/").split("/")[-1]
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(a.xpath(".//img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = cls._first_str(a.xpath("./@title"))
+
+            word_count = cls._first_str(a.xpath(".//eps//text()")) or "-"
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author="-",
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/esjzone.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.esjzone
+---------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["esjzone"],
+)
+class EsjzoneSearcher(BaseSearcher):
+    site_name = "esjzone"
+    priority = 30
+    BASE_URL = "https://www.esjzone.cc"
+    SEARCH_URL = "https://www.esjzone.cc/tags/{query}/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        url = cls.SEARCH_URL.format(query=cls._quote(keyword))
+        try:
+            async with (await cls._http_get(url)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                url,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        cards = doc.xpath('//div[contains(@class,"card-body")]')
+        results: list[SearchResult] = []
+
+        for idx, card in enumerate(cards):
+            href = cls._first_str(
+                card.xpath(".//h5[contains(@class,'card-title')]/a[1]/@href")
+            )
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # href format: /detail/<book_id>.html
+            book_id = href.split("/")[-1].split(".")[0]
+            book_url = cls._abs_url(href)
+
+            title = cls._first_str(
+                card.xpath(".//h5[contains(@class,'card-title')]/a[1]//text()")
+            )
+
+            latest_chapter = (
+                cls._first_str(
+                    card.xpath(".//div[contains(@class,'card-ep')]//a[1]//text()")
+                )
+                or "-"
+            )
+
+            # Author
+            author = cls._first_str(
+                card.xpath(".//div[contains(@class,'card-author')]//a[1]//text()")
+            ) or cls._first_str(
+                card.xpath(".//div[contains(@class,'card-author')]//text()")
+            )
+
+            cover_data = card.xpath(
+                './preceding-sibling::a[contains(@class,"card-img-tiles")]'
+                '//div[contains(@class,"lazyload")]/@data-src'
+            )
+            cover_url = cover_data[0].strip() if cover_data else ""
+
+            # Compute priority incrementally
+            prio = cls.priority + idx
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/hetushu.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.hetushu
+---------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["hetushu"],
+)
+class HetushuSearcher(BaseSearcher):
+    site_name = "hetushu"
+    priority = 5
+    SEARCH_URL = "https://www.hetushu.com/search/"
+    BASE_URL = "https://www.hetushu.com"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"keyword": keyword}
+        headers = {
+            "Referer": "https://www.hetushu.com/",
+        }
+        try:
+            async with (
+                await cls._http_get(cls.SEARCH_URL, params=params, headers=headers)
+            ) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath('//dl[@class="list" and @id="body"]/dd')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//h4/a/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # "/book/7631/index.html" -> "7631"
+            book_id = href.rstrip("/index.html").split("/")[-1]
+            book_url = cls._abs_url(href)
+
+            title = cls._first_str(row.xpath(".//h4/a/text()"))
+
+            # Author from the adjacent <span>, strip "/" delimiters
+            # e.x. " / 风行云亦行 / "
+            author_raw = cls._first_str(row.xpath(".//h4/span/text()"))
+            author = author_raw.strip("/").strip()
+
+            cover_rel = cls._first_str(row.xpath(".//a/img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/i25zw.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.i25zw
+-------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["i25zw"],
+)
+class I25zwSearcher(BaseSearcher):
+    site_name = "i25zw"
+    priority = 30
+    SEARCH_URL = "https://www.i25zw.com/search.html"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        payload = {
+            "searchkey": keyword,
+            "searchtype": "all",
+            "Submit": "",
+        }
+        try:
+            async with (await cls._http_post(cls.SEARCH_URL, data=payload)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath("//div[@id='alistbox']")
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            book_url = cls._first_str(row.xpath(".//div[@class='pic']/a/@href"))
+            if not book_url:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # 'https://www.i25zw.com/book/309209.html' -> "309209"
+            book_id = book_url.split("/")[-1].split(".")[0]
+
+            title = cls._first_str(row.xpath(".//div[@class='title']/h2/a/text()"))
+
+            author = cls._first_str(
+                row.xpath(".//div[@class='title']/span/text()"),
+                replaces=[("作者：", "")],
+            )
+
+            cover_rel = cls._first_str(row.xpath(".//div[@class='pic']//img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            # Latest chapter
+            latest_chapter = (
+                cls._first_str(row.xpath(".//div[@class='sys']//li[1]/a/text()")) or "-"
+            )
+
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results