novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/searchers/quanben5.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.quanben5
+----------------------------------------
+
+"""
+
+import json
+import logging
+import random
+import time
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["quanben5"],
+)
+class Quanben5Searcher(BaseSearcher):
+    site_name = "quanben5"
+    priority = 30
+    BASE_URL = "https://quanben5.com"
+    SEARCH_URL = "https://quanben5.com/"
+
+    STATIC_CHARS = "PXhw7UT1B0a9kQDKZsjIASmOezxYG4CHo5Jyfg2b8FLpEvRr3WtVnlqMidu6cN"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        t = str(int(time.time() * 1000))
+        uri_keyword = cls._quote(keyword)
+        b_raw = cls._base64(uri_keyword)
+        b = cls._quote(b_raw)
+
+        params = {
+            "c": "book",
+            "a": "search.json",
+            "callback": "search",
+            "t": t,
+            "keywords": uri_keyword,
+            "b": b,
+        }
+        full_url = cls._build_url(cls.SEARCH_URL, params)
+
+        headers = {
+            "Host": "quanben5.com",
+            "Referer": "https://quanben5.com/search.html",
+        }
+
+        try:
+            async with (await cls._http_get(full_url, headers=headers)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        # Unwrap JSONP: search({...});
+        prefix, suffix = "search(", ");"
+        json_str = (
+            html_str[len(prefix) : -len(suffix)]
+            if html_str.startswith(prefix) and html_str.endswith(suffix)
+            else html_str
+        )
+
+        try:
+            data = json.loads(json_str)
+        except json.JSONDecodeError:
+            return []
+
+        content_html = data.get("content", "")
+        if not content_html:
+            return []
+
+        doc = html.fromstring(content_html)
+        rows = doc.xpath('//div[@class="pic_txt_list"]')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//h3/a/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # '/n/douposanqian/' -> "douposanqian"
+            book_id = href.rstrip("/").split("/")[-1]
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(row.xpath(".//div[@class='pic']//img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = "".join(
+                t.strip()
+                for t in row.xpath(".//h3/a/span[@class='name']//text()")
+                if t and t.strip()
+            )
+
+            author = cls._first_str(
+                row.xpath(".//p[@class='info']//span[contains(@class,'author')]/text()")
+            )
+
+            # Bump priority by result index
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results
+
+    @classmethod
+    def _base64(cls, s: str) -> str:
+        out = []
+        for ch in s:
+            idx = cls.STATIC_CHARS.find(ch)
+            code = cls.STATIC_CHARS[(idx + 3) % 62] if idx != -1 else ch
+            n1 = int(random.random() * 62)
+            n2 = int(random.random() * 62)
+            out.append(cls.STATIC_CHARS[n1])
+            out.append(code)
+            out.append(cls.STATIC_CHARS[n2])
+        return "".join(out)

novel_downloader/core/searchers/registry.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.registry
+----------------------------------------
+
+"""
+
+__all__ = ["register_searcher", "search"]
+
+import asyncio
+from collections.abc import Callable, Sequence
+from typing import TypeVar
+
+import aiohttp
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+S = TypeVar("S", bound=BaseSearcher)
+
+_SEARCHER_REGISTRY: dict[str, type[BaseSearcher]] = {}
+
+
+def register_searcher(
+    site_keys: Sequence[str],
+) -> Callable[[type[S]], type[S]]:
+    """
+    Decorator to register a searcher class under given name.
+    """
+
+    def decorator(cls: type[S]) -> type[S]:
+        for key in site_keys:
+            _SEARCHER_REGISTRY[key] = cls
+        return cls
+
+    return decorator
+
+
+async def search(
+    keyword: str,
+    sites: Sequence[str] | None = None,
+    limit: int | None = None,
+    per_site_limit: int = 5,
+    timeout: float = 5.0,
+) -> list[SearchResult]:
+    """
+    Perform a search for the given keyword across one or more registered sites,
+    then aggregate and sort the results by their `priority` value.
+
+    :param keyword: The search term or keyword to query.
+    :param sites: An optional sequence of site keys to limit which searchers.
+    :param limit: Maximum total number of results to return; if None, return all.
+    :param per_site_limit: Maximum number of search results per site.
+    :param timeout: Per-request time budget (seconds)
+    :return: A flat list of `SearchResult` objects.
+    """
+    keys = list(sites or _SEARCHER_REGISTRY.keys())
+    to_call = {_SEARCHER_REGISTRY[key] for key in keys if key in _SEARCHER_REGISTRY}
+
+    site_timeout = aiohttp.ClientTimeout(total=timeout)
+
+    results: list[SearchResult] = []
+    async with aiohttp.ClientSession(timeout=site_timeout) as session:
+        # Give all searchers the same session
+        for cls in to_call:
+            cls.configure(session)
+
+        # Kick off all sites in parallel
+        coros = [cls.search(keyword, limit=per_site_limit) for cls in to_call]
+        site_lists = await asyncio.gather(*coros, return_exceptions=True)
+
+    # Collect successful results; skip failures
+    for item in site_lists:
+        if isinstance(item, Exception | BaseException):
+            continue
+        results.extend(item)
+
+    results.sort(key=lambda res: res["priority"])
+    return results[:limit] if limit is not None else results

novel_downloader/core/searchers/shuhaige.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.shuhaige
+----------------------------------------
+
+"""
+
+import logging
+import time
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["shuhaige"],
+)
+class ShuhaigeSearcher(BaseSearcher):
+    site_name = "shuhaige"
+    priority = 30
+    BASE_URL = "https://www.shuhaige.net"
+    SEARCH_URL = "https://www.shuhaige.net/search.html"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        data = {
+            "searchtype": "all",
+            "searchkey": keyword,
+        }
+        ts = int(time.time())
+        # baidu cookie format: f"Hm_lpvt_{site_id}={timestamp}"
+        cookie_str = (
+            f"Hm_lpvt_3094b20ed277f38e8f9ac2b2b29d6263={ts}; "
+            f"Hm_lpvt_c3da01855456ad902664af23cc3254cb={ts}"
+        )
+        headers = {
+            "Origin": "https://www.shuhaige.net",
+            "Referer": "https://www.shuhaige.net/",
+            "Cookie": cookie_str,
+        }
+        try:
+            async with (
+                await cls._http_post(cls.SEARCH_URL, data=data, headers=headers)
+            ) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath('//div[@id="sitembox"]/dl')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath("./dt/a[1]/@href")) or cls._first_str(
+                row.xpath("./dd/h3/a[1]/@href")
+            )
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            book_id = href.strip("/").split("/")[0]
+            book_url = cls._abs_url(href)
+
+            title = cls._first_str(row.xpath("./dd/h3/a[1]//text()")) or cls._first_str(
+                row.xpath("./dt/a[1]/img[1]/@alt")
+            )
+
+            cover_rel = cls._first_str(row.xpath("./dt/a[1]/img[1]/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            author = (
+                cls._first_str(row.xpath("./dd[@class='book_other'][1]/span[1]/text()"))
+                or "-"
+            )
+            word_count = (
+                cls._first_str(row.xpath("./dd[@class='book_other'][1]/span[4]/text()"))
+                or "-"
+            )
+
+            latest_chapter = (
+                cls._first_str(
+                    row.xpath("./dd[@class='book_other'][last()]/a[1]//text()")
+                )
+                or "-"
+            )
+            update_date = (
+                cls._first_str(
+                    row.xpath("./dd[@class='book_other'][last()]/span[1]//text()")
+                )
+                or "-"
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/tongrenquan.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.tongrenquan
+-------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["tongrenquan"],
+)
+class TongrenquanSearcher(BaseSearcher):
+    site_name = "tongrenquan"
+    priority = 30
+    SEARCH_URL = "https://www.tongrenquan.org/e/search/indexstart.php"
+    BASE_URL = "https://www.tongrenquan.org"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        keyboard = cls._quote(keyword, encoding="gbk", errors="replace")
+        show = "title"
+        classid = "0"
+        body = f"keyboard={keyboard}&show={show}&classid={classid}"
+        headers = {
+            "Origin": "https://www.tongrenquan.cc",
+            "Referer": "https://www.tongrenquan.cc/",
+            "Content-Type": "application/x-www-form-urlencoded",
+        }
+        try:
+            async with (
+                await cls._http_post(cls.SEARCH_URL, data=body, headers=headers)
+            ) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath('//div[@class="books m-cols"]/div[@class="bk"]')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//h3/a[1]/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # '/tongren/9302.html' -> "9302"
+            book_id = href.split("/")[-1].split(".")[0]
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(
+                row.xpath("./div[@class='pic']/a[1]/img[1]/@src")
+            )
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = cls._first_str(
+                row.xpath("./div[@class='bk_right']/h3/a[1]//text()")
+            )
+
+            author = (
+                cls._first_str(
+                    row.xpath("./div[@class='bk_right']/div[@class='booknews']/text()"),
+                    replaces=[("作者：", "")],
+                )
+                or "-"
+            )
+
+            update_date = cls._first_str(
+                row.xpath(
+                    "./div[@class='bk_right']/div[@class='booknews']/label[@class='date']/text()"
+                )
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date=update_date,
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/ttkan.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.ttkan
+-------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["ttkan"],
+)
+class TtkanSearcher(BaseSearcher):
+    site_name = "ttkan"
+    priority = 100
+    BASE_URL = "https://www.ttkan.co"
+    SEARCH_URL = "https://www.ttkan.co/novel/search"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"q": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        items = doc.xpath(
+            '//div[contains(@class,"frame_body")]//div[@class="pure-g"]/div[contains(@class,"novel_cell")]'
+        )
+        if not items:
+            items = doc.xpath('//div[contains(@class,"novel_cell")]')
+        results: list[SearchResult] = []
+
+        for idx, item in enumerate(items):
+            href = cls._first_str(item.xpath(".//a[@href][1]/@href"))
+            if not href:
+                continue
+
+            if limit is not None and len(results) >= limit:
+                break
+
+            # link -> /novel/chapters/<book_id>
+            book_id = href.strip("/").split("/")[-1]
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(item.xpath(".//amp-img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = cls._first_str(item.xpath(".//h3/text()"))
+
+            author = (
+                cls._first_str(
+                    item.xpath(".//li[starts-with(normalize-space(.),'作者')]/text()"),
+                    replaces=[("作者：", "")],
+                )
+                or "-"
+            )
+
+            prio = cls.priority + idx
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/xiaoshuowu.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.xiaoshuowu
+------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["xiaoshuowu", "xiaoshuoge"],
+)
+class XiaoshuowuSearcher(BaseSearcher):
+    site_name = "xiaoshuowu"
+    priority = 30
+    SEARCH_URL = "http://www.xiaoshuoge.info/modules/article/search.php"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"q": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        """
+        Parse raw HTML from Xiaoshuowu search results into list of SearchResult.
+
+        :param html_str: Raw HTML string from Xiaoshuowu search results page.
+        :param limit: Maximum number of results to return, or None for all.
+        :return: List of SearchResult dicts.
+        """
+        doc = html.fromstring(html_str)
+        rows = doc.xpath('//div[@class="c_row"]')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//span[@class='c_subject']/a/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            # 'http://www.xiaoshuoge.info/book/374339/' -> "374339"
+            book_id = href.split("book/")[-1].strip("/")
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(row.xpath(".//div[@class='fl']//img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = cls._first_str(row.xpath(".//span[@class='c_subject']/a/text()"))
+
+            author = (
+                cls._first_str(
+                    row.xpath(
+                        ".//div[@class='c_tag'][1]/span[@class='c_label'][contains(.,'作者')]/following-sibling::span[@class='c_value'][1]/text()"
+                    )
+                )
+                or "-"
+            )
+            word_count = (
+                cls._first_str(
+                    row.xpath(
+                        ".//div[@class='c_tag'][1]/span[@class='c_label'][contains(.,'字数')]/following-sibling::span[@class='c_value'][1]/text()"
+                    )
+                )
+                or "-"
+            )
+
+            latest_chapter = (
+                cls._first_str(
+                    row.xpath(
+                        ".//div[@class='c_tag'][last()]/span[@class='c_label'][contains(.,'最新')]/following-sibling::span[@class='c_value'][1]//a//text()"
+                    )
+                )
+                or "-"
+            )
+            update_date = (
+                cls._first_str(
+                    row.xpath(
+                        ".//div[@class='c_tag'][last()]/span[@class='c_label'][contains(.,'更新')]/following-sibling::span[@class='c_value'][1]/text()"
+                    )
+                )
+                or "-"
+            )
+
+            # Priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results