novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/archived/deqixs/fetcher.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.fetcher
+---------------------------------------------
+
+"""
+
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.models import FetcherConfig
+from novel_downloader.utils import async_jitter_sleep
+
+# from novel_downloader.core.fetchers.registry import register_fetcher
+
+
+# @register_fetcher(
+#     site_keys=["deqixs"],
+# )
+class DeqixsSession(BaseSession):
+    """
+    A session class for interacting with the 得奇小说网 (www.deqixs.com) novel website.
+    """
+
+    BASE_URL = "https://www.deqixs.com"
+    BOOK_INFO_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/"
+    CHAPTER_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("deqixs", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as a string.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The chapter content as a string.
+        """
+        html_pages: list[str] = []
+        idx = 1
+
+        while True:
+            chapter_suffix = chapter_id if idx == 1 else f"{chapter_id}-{idx}"
+            relative_path = f"/xiaoshuo/{book_id}/{chapter_suffix}.html"
+            full_url = self.BASE_URL + relative_path
+
+            if idx > 1 and relative_path not in html_pages[-1]:
+                break
+
+            try:
+                html = await self.fetch(full_url, **kwargs)
+            except Exception as exc:
+                self.logger.warning(
+                    "[async] get_book_chapter(%s page %d) failed: %s",
+                    chapter_id,
+                    idx,
+                    exc,
+                )
+                break
+
+            html_pages.append(html)
+            idx += 1
+            await async_jitter_sleep(
+                self.request_interval,
+                mul_spread=1.1,
+                max_sleep=self.request_interval + 2,
+            )
+
+        return html_pages
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)

novel_downloader/core/archived/deqixs/parser.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.parser
+--------------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+# from novel_downloader.core.parsers.registry import register_parser
+
+
+# @register_parser(
+#     site_keys=["deqixs"],
+# )
+class DeqixsParser(BaseParser):
+    """
+    Parser for 得奇小说网 book pages.
+    """
+
+    ADS: set[str] = {
+        "更新不易",
+        "记得分享",
+        "(本章完)",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # Extract book title and word count
+        book_name = tree.xpath("//div[@class='itemtxt']/h1/a/text()")[0].strip()
+        word_count = tree.xpath("//div[@class='itemtxt']/h1/i/text()")[0].strip()
+
+        # Extract serialization status and genre tags
+        spans = tree.xpath("//div[@class='itemtxt']/p[1]/span/text()")
+        serial_status = spans[0].strip() if spans else ""
+        tags = [s.strip() for s in spans[1:-1]] if len(spans) > 2 else []
+
+        # Extract author
+        author_text = tree.xpath("//div[@class='itemtxt']/p[2]/a/text()")[0]
+        author = author_text.replace("作者：", "").strip()
+
+        # Extract cover URL
+        cover_src = tree.xpath("//div[@class='item']//a/img/@src")[0]
+        cover_url = "https:" + cover_src if cover_src.startswith("//") else cover_src
+
+        # Extract last update time
+        update_raw = tree.xpath("//h2[@id='dir']/span/text()")[0].strip()
+        update_time = update_raw.replace("更新时间：", "").strip()
+
+        # Extract summary paragraphs (first description block)
+        paras = tree.xpath("(//div[@class='des bb'])[1]/p/text()")
+        summary = "\n".join(p.strip() for p in paras if p.strip())
+
+        # Extract chapters list
+        chapter_nodes = tree.xpath("//div[@id='list']//ul/li/a")
+        chapters: list[ChapterInfoDict] = []
+        for a in chapter_nodes:
+            href = a.get("href")
+            chapter_id = href.split("/")[-1].replace(".html", "")
+            title = a.text_content().strip()
+            chapters.append({"title": title, "url": href, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "serial_status": serial_status,
+            "word_count": word_count,
+            "summary": summary,
+            "tags": tags,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        title_text = ""
+        contents: list[str] = []
+        for curr_html in html_list:
+            tree = html.fromstring(curr_html)
+            # Extract title once
+            if not title_text:
+                full_title = tree.xpath("string(//div[@class='submenu']/h1)")
+                if ">" in full_title:
+                    title_text = full_title.split(">", 1)[1].strip()
+                else:
+                    title_text = full_title.strip()
+            # Extract paragraphs
+            for p in tree.xpath("//div[@class='con']/p"):
+                text = p.text_content().strip()
+                # Filter out ads or empty paragraphs
+                if not text or any(ad in text for ad in self.ADS):
+                    continue
+                contents.append(text)
+
+        content = "\n".join(contents)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title_text,
+            "content": content,
+            "extra": {"site": "deqixs"},
+        }

novel_downloader/core/archived/deqixs/searcher.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.searcher
+----------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["deqixs"],
+# )
+class DeqixsSearcher(BaseSearcher):
+    site_name = "deqixs"
+    priority = 20
+    BASE_URL = "https://www.deqixs.com"
+    SEARCH_URL = "https://www.deqixs.com/tag/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"key": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath("//div[@class='container']/div[@class='item']")
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+
+            href = row.xpath(".//h3/a/@href")[0]
+            book_id = href.strip("/ ").split("/")[-1]
+            if not book_id:
+                continue
+            book_url = cls.BASE_URL + href
+            img_src = row.xpath(".//a/img/@src")[0]
+            cover_url = "https:" + img_src if img_src.startswith("//") else img_src
+            title = row.xpath(".//h3/a/text()")[0].strip()
+
+            author_text = row.xpath(".//p[2]/a/text()")[0]
+            author = author_text.replace("作者：", "").strip()
+
+            spans = row.xpath(".//p[1]/span/text()")
+            word_count = spans[2].strip() if len(spans) > 2 else ""
+
+            # Extract latest chapter and update date
+            first_li = row.xpath(".//ul/li")[0]
+            update_date = first_li.xpath("./i/text()")[0].strip()
+            latest_chapter = first_li.xpath("./a/text()")[0].strip()
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/archived/qidian/searcher.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.qidian.searcher
+----------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["qidian", "qd"],
+# )
+class QidianSearcher(BaseSearcher):
+    """
+    TODO: 现在默认没有 cookie 会跳转
+    """
+
+    site_name = "qidian"
+    priority = 0
+    SEARCH_URL = "https://www.qidian.com/so/{query}.html"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        url = cls.SEARCH_URL.format(query=cls._quote(keyword))
+        try:
+            async with (await cls._http_get(url)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                url,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        items = doc.xpath(
+            '//div[@id="result-list"]//li[contains(@class, "res-book-item")]'
+        )
+        results: list[SearchResult] = []
+
+        base_prio = getattr(cls, "priority", 0)
+        for idx, item in enumerate(items):
+            if limit is not None and idx >= limit:
+                break
+            book_id = item.get("data-bid")
+            if not book_id:
+                continue
+            title_elem = item.xpath('.//h3[@class="book-info-title"]/a')[0]
+            title = title_elem.text_content().strip()
+            author_nodes = item.xpath(
+                './/p[@class="author"]/a[@class="name"] | .//p[@class="author"]/i'
+            )
+            author = author_nodes[0].text_content().strip() if author_nodes else ""
+            prio = base_prio + idx
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url="",
+                    cover_url="",
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/archived/wanbengo/searcher.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.wanbengo.searcher
+------------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["wanbengo"],
+# )
+class WanbengoSearcher(BaseSearcher):
+    site_name = "wanbengo"
+    priority = 30
+    BASE_URL = "https://www.wanbengo.com"
+    SEARCH_URL = "https://www.sososhu.com/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {
+            "q": keyword,
+            "site": "wbsz",
+        }
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath(
+            "//div[contains(@class,'so_list')]//div[contains(@class,'hot')]//div[contains(@class,'item')]"
+        )
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+            a_nodes = row.xpath(".//dl/dt/a[1]")
+            a = a_nodes[0] if a_nodes else None
+            href = a.get("href") if a is not None else ""
+            if not href:
+                continue
+
+            book_url = cls._restore_url(cls._abs_url(href))
+            book_id = cls._book_id_from_url(book_url) if book_url else ""
+
+            title = (a.text_content() if a is not None else "").strip()
+            author = cls._first_str(row.xpath(".//dl/dt/span[1]/text()"))
+            cover_url = cls._first_str(
+                row.xpath(".//div[contains(@class,'image')]//img/@src")
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results
+
+    @staticmethod
+    def _restore_url(url: str) -> str:
+        return url.replace("www.wbsz.org", "www.wanbengo.com")
+
+    @staticmethod
+    def _book_id_from_url(url: str) -> str:
+        tail = url.split("wanbengo.com", 1)[-1]
+        tail = tail.strip("/")
+        return tail.replace("/", "-")

novel_downloader/core/archived/xshbook/searcher.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.xshbook.searcher
+-----------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["xshbook"],
+# )
+class XshbookSearcher(BaseSearcher):
+    site_name = "xshbook"
+    priority = 30
+    BASE_URL = "https://www.xshbook.com"
+    SEARCH_URL = "https://www.sososhu.com/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {
+            "q": keyword,
+            "site": "xshbook",
+        }
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath(
+            "//div[contains(@class,'so_list')]//div[contains(@class,'hot')]//div[contains(@class,'item')]"
+        )
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+            a_nodes = row.xpath(".//dl/dt/a[1]")
+            a = a_nodes[0] if a_nodes else None
+            href = a.get("href") if a is not None else ""
+            book_url = cls._abs_url(href)
+            book_id = cls._book_id_from_url(book_url) if book_url else ""
+            if not book_id:
+                continue
+
+            title = (a.text_content() if a is not None else "").strip()
+            author = cls._first_str(row.xpath(".//dl/dt/span[1]/text()"))
+            cover_url = cls._first_str(
+                row.xpath(".//div[contains(@class,'image')]//img/@src")
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results
+
+    @staticmethod
+    def _book_id_from_url(url: str) -> str:
+        tail = url.split("xshbook.com", 1)[-1]
+        tail = tail.strip("/")
+        return tail.replace("/", "-")

novel_downloader/core/downloaders/__init__.py

@@ -3,39 +3,17 @@
 novel_downloader.core.downloaders
 ---------------------------------
 
-This subpackage contains concrete downloader implementations for
-specific novel platforms.
-
-Each downloader is responsible for orchestrating the full lifecycle
-of retrieving, parsing, and saving novel content for a given source.
-
-Currently supported platforms:
-- biquge (笔趣阁)
-- esjzone (ESJ Zone)
-- linovelib (哔哩轻小说)
-- qianbi (铅笔小说)
-- qidian (起点中文网)
-- sfacg (SF轻小说)
-- yamibo (百合会)
-- common (通用架构)
+Downloader implementations for retrieving novels from different sources
 """
 
-from .biquge import BiqugeDownloader
-from .common import CommonDownloader
-from .esjzone import EsjzoneDownloader
-from .linovelib import LinovelibDownloader
-from .qianbi import QianbiDownloader
-from .qidian import QidianDownloader
-from .sfacg import SfacgDownloader
-from .yamibo import YamiboDownloader
-
 __all__ = [
-    "BiqugeDownloader",
-    "EsjzoneDownloader",
-    "LinovelibDownloader",
+    "get_downloader",
+    "CommonDownloader",
     "QianbiDownloader",
     "QidianDownloader",
-    "SfacgDownloader",
-    "YamiboDownloader",
-    "CommonDownloader",
 ]
+
+from .common import CommonDownloader
+from .qianbi import QianbiDownloader
+from .qidian import QidianDownloader
+from .registry import get_downloader