novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/xshbook.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.xshbook
+-------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["xshbook"],
+)
+class XshbookParser(BaseParser):
+    """Parser for 小说虎 book pages."""
+
+    BASE = "http://www.xshbook.com"
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath("//div[@id='info']/h1/text()"))
+
+        author = self._first_str(
+            tree.xpath("//div[@id='info']/p[1]/text()"),
+            replaces=[("\xa0", ""), ("作者:", "")],
+        )
+
+        update_time = self._first_str(
+            tree.xpath("//meta[@property='og:novel:update_time']/@content")
+        )
+
+        summary = "\n".join(
+            self._first_str(p.xpath("string()").splitlines())
+            for p in tree.xpath("//div[@id='intro']//p")
+        ).strip()
+        summary = summary.split("本站提示", 1)[0].strip()
+
+        cover_url = self._first_str(tree.xpath("//div[@id='fmimg']//img/@src"))
+
+        book_type = self._first_str(tree.xpath("//div[@class='con_top']/a[2]/text()"))
+        tags: list[str] = [book_type] if book_type else []
+
+        chapters: list[ChapterInfoDict] = []
+        for a in tree.xpath("//div[@id='list']//dd/a"):
+            href = a.get("href", "")
+            title = self._norm_space(a.text_content())
+            # /95071/95071941/389027455.html -> "389027455"
+            chapter_id = href.rsplit("/", 1)[-1].split(".", 1)[0]
+            chapters.append({"title": title, "url": href, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "summary": summary,
+            "tags": tags,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath("//div[@class='bookname']/h1/text()"))
+        if not title:
+            title = self._first_str(
+                tree.xpath("//div[@class='con_top']/text()[last()]")
+            )
+
+        cont_nodes = tree.xpath("//div[@id='content']")
+        if not cont_nodes:
+            return None
+        cont = cont_nodes[0]
+
+        # remove scripts under content
+        for s in cont.xpath(".//script"):
+            s.getparent().remove(s)
+
+        paragraphs: list[str] = []
+        for p in cont.xpath(".//p"):
+            text = html.tostring(p, method="text", encoding="unicode")
+            text = text.replace("\xa0", " ")
+            # filter boilerplate lines
+            bad = (
+                "谨记我们的网址" in text
+                or "温馨提示" in text
+                or "提示" in text
+                and "本文" not in text
+                and len(text) < 60
+                or "分享" in text
+                and len(text) < 40
+            )
+            if not bad:
+                paragraphs.append(text)
+
+        content = "\n".join(self._norm_space(p) for p in paragraphs if p.strip())
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "xshbook"},
+        }

novel_downloader/core/parsers/yamibo.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.yamibo
+------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["yamibo"],
+)
+class YamiboParser(BaseParser):
+    """
+    Parser for 百合会 book pages.
+    """
+
+    BASE_URL = "https://www.yamibo.com"
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(
+            tree.xpath('//h3[contains(@class,"col-md-12")]/text()')
+        )
+        author = self._first_str(
+            tree.xpath('//h5[contains(@class,"text-warning")]/text()')
+        )
+        cover_url = self.BASE_URL + self._first_str(
+            tree.xpath('//img[contains(@class,"img-responsive")]/@src')
+        )
+
+        update_time = self._first_str(
+            tree.xpath('//p[contains(text(),"更新时间：")]/text()'),
+            replaces=[("更新时间：", "")],
+        )
+        serial_status = self._first_str(
+            tree.xpath('//p[contains(text(),"作品状态：")]/text()'),
+            replaces=[("作品状态：", "")],
+        )
+        book_type = self._first_str(
+            tree.xpath('//p[contains(text(),"作品分类：")]/text()'),
+            replaces=[("作品分类：", "")],
+        )
+        summary = self._first_str([tree.xpath('string(//div[@id="w0-collapse1"]/div)')])
+
+        # volumes & chapters
+        volumes: list[VolumeInfoDict] = []
+        for volume_node in tree.xpath(
+            '//div[contains(@class,"panel-info") and contains(@class,"panel-default")]'
+        ):
+            volume_name = (
+                self._first_str(
+                    volume_node.xpath(
+                        './/div[contains(@class,"panel-heading")]//a/text()'
+                    )
+                )
+                or "未命名卷"
+            )
+            chapters: list[ChapterInfoDict] = []
+            for chap in volume_node.xpath(
+                './/div[contains(@class,"panel-body")]//a[contains(@href,"view-chapter")]'
+            ):
+                title = self._first_str([chap.xpath("string()")])
+                url = chap.get("href", "")
+                chapter_id = url.split("id=")[-1]
+                chapters.append({"title": title, "url": url, "chapterId": chapter_id})
+            volumes.append({"volume_name": volume_name, "chapters": chapters})
+
+        # fallback: flat chapter list
+        if not volumes:
+            chapters = []
+            for chap in tree.xpath(
+                '//div[@class="panel-body"]//a[contains(@href,"view-chapter")]'
+            ):
+                title = self._first_str([chap.xpath("string()")])
+                url = chap.get("href", "")
+                chapter_id = url.split("id=")[-1] if "id=" in url else ""
+                chapters.append({"title": title, "url": url, "chapterId": chapter_id})
+            volumes = [{"volume_name": "单卷", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "serial_status": serial_status,
+            "tags": [book_type],
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        content_lines = tree.xpath("//div[@id='w0-collapse1']//p//text()")
+        content = "\n".join(line.strip() for line in content_lines if line.strip())
+        if not content:
+            return None
+
+        title = self._first_str(
+            [tree.xpath("string(//section[contains(@class,'col-md-9')]//h3)")]
+        )
+
+        updated_at = self._first_str(
+            tree.xpath(
+                "//div[contains(@class,'row')]//div[contains(text(),'更新时间')]/text()"
+            ),
+            replaces=[("更新时间：", "")],
+        )
+        word_str = self._first_str(
+            tree.xpath(
+                "//div[contains(@class,'row')]//div[contains(text(),'章节字数')]/text()"
+            ),
+            replaces=[("章节字数：", "")],
+        )
+        word_count = int(word_str) if word_str.isdigit() else 0
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {
+                "site": "yamibo",
+                "word_count": word_count,
+                "updated_at": updated_at,
+            },
+        }

novel_downloader/core/parsers/yibige.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.yibige
+------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["yibige"],
+)
+class YibigeParser(BaseParser):
+    """
+    Parser for 一笔阁 book pages.
+    """
+
+    ADS = {
+        "首发无广告",
+        "请分享",
+        "读之阁",
+        "小说网",
+        "首发地址",
+        "手机阅读",
+        "一笔阁",
+        "site_con_ad(",
+        "chapter_content(",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if len(html_list) < 2:
+            return None
+
+        # Parse trees
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
+
+        # --- From <meta> data ---
+        book_name = self._meta(info_tree, "og:novel:book_name") or self._first_str(
+            info_tree.xpath("//div[@id='info']/h1/text()")
+        )
+
+        author = self._meta(info_tree, "og:novel:author") or self._first_str(
+            info_tree.xpath("//div[@id='info']/p[a]/a/text()")
+        )
+
+        cover_url = self._meta(info_tree, "og:image") or self._first_str(
+            info_tree.xpath("//div[@id='fmimg']//img/@src")
+        )
+
+        update_time = self._meta(info_tree, "og:novel:update_time").replace("T", " ")
+        serial_status = self._meta(info_tree, "og:novel:status") or "连载中"
+
+        word_count = self._first_str(
+            info_tree.xpath("//div[@id='info']/p[contains(., '字数：')]/text()[1]"),
+            replaces=[("字数：", "")],
+        )
+
+        # Summary: first paragraph under #intro
+        summary = self._first_str(info_tree.xpath("//div[@id='intro']//p[1]/text()"))
+
+        # Category and tags
+        book_type = self._meta(info_tree, "og:novel:category")
+        tags_set = set(self._meta_all(info_tree, "book:tag"))
+        if book_type:
+            tags_set.add(book_type)
+        tags = list(tags_set)
+
+        # --- Chapters from the catalog page ---
+        chapters: list[ChapterInfoDict] = []
+        for a in catalog_tree.xpath("//div[@id='list']/dl/dd/a"):
+            href = (a.get("href") or "").strip()
+            if not href:
+                continue
+            title = (a.text_content() or "").strip()
+            if not title:
+                continue
+            # /6238/2496.html -> 2496
+            chap_id = href.split("/")[-1].split(".")[0]
+            chapters.append({"title": title, "url": href, "chapterId": chap_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "serial_status": serial_status,
+            "word_count": word_count,
+            "summary": summary,
+            "tags": tags,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath("//div[@class='bookname']/h1/text()"))
+
+        paragraphs: list[str] = []
+        for p in tree.xpath("//div[@id='content']//p"):
+            txt = self._norm_space(p.text_content())
+            if not txt or self._is_ad(txt):
+                continue
+            paragraphs.append(txt)
+
+        content = "\n".join(paragraphs).strip()
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "yibige"},
+        }
+
+    def _is_ad(self, s: str) -> bool:
+        """
+        Filter for footer junk inside #content.
+        """
+        if self._is_ad_line(s):
+            return True
+
+        ss = s.replace(" ", "")
+        # return any(b in s or b in ss for b in self.ADS)
+        return self._is_ad_line(ss)
+
+    @classmethod
+    def _meta(cls, tree: html.HtmlElement, prop: str) -> str:
+        """
+        Get a single meta property content
+        """
+        return cls._first_str(tree.xpath(f"//meta[@property='{prop}']/@content"))
+
+    @staticmethod
+    def _meta_all(tree: html.HtmlElement, prop: str) -> list[str]:
+        """
+        Get all meta property content values
+        """
+        return tree.xpath(f"//meta[@property='{prop}']/@content") or []

novel_downloader/core/searchers/__init__.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers
+-------------------------------
+
+Site-specific searcher implementations for discovering novels across multiple sources
+"""
+
+__all__ = [
+    "search",
+    "AaatxtSearcher",
+    "BiqugeSearcher",
+    "DxmwxSearcher",
+    "EightnovelSearcher",
+    "EsjzoneSearcher",
+    "HetushuSearcher",
+    "I25zwSearcher",
+    "Ixdzs8Searcher",
+    "Jpxs123Searcher",
+    "PiaotiaSearcher",
+    "QbtrSearcher",
+    "QianbiSearcher",
+    "Quanben5Searcher",
+    "ShuhaigeSearcher",
+    "TongrenquanSearcher",
+    "TtkanSearcher",
+    "XiaoshuowuSearcher",
+    "XiguashuwuSearcher",
+    "Xs63bSearcher",
+]
+
+from .aaatxt import AaatxtSearcher
+from .b520 import BiqugeSearcher
+from .dxmwx import DxmwxSearcher
+from .eightnovel import EightnovelSearcher
+from .esjzone import EsjzoneSearcher
+from .hetushu import HetushuSearcher
+from .i25zw import I25zwSearcher
+from .ixdzs8 import Ixdzs8Searcher
+from .jpxs123 import Jpxs123Searcher
+from .piaotia import PiaotiaSearcher
+from .qbtr import QbtrSearcher
+from .qianbi import QianbiSearcher
+from .quanben5 import Quanben5Searcher
+from .registry import search
+from .shuhaige import ShuhaigeSearcher
+from .tongrenquan import TongrenquanSearcher
+from .ttkan import TtkanSearcher
+from .xiaoshuowu import XiaoshuowuSearcher
+from .xiguashuwu import XiguashuwuSearcher
+from .xs63b import Xs63bSearcher

novel_downloader/core/searchers/aaatxt.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.aaatxt
+--------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["aaatxt"],
+)
+class AaatxtSearcher(BaseSearcher):
+    site_name = "aaatxt"
+    priority = 500
+    SEARCH_URL = "http://www.aaatxt.com/search.php"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        # gbk / gb2312
+        params = {
+            "keyword": cls._quote(keyword, encoding="gb2312", errors="replace"),
+            "submit": cls._quote("搜 索", encoding="gb2312", errors="replace"),
+        }
+        full_url = cls._build_url(cls.SEARCH_URL, params)  # need build manually
+        headers = {
+            "Host": "www.aaatxt.com",
+            "Referer": "http://www.aaatxt.com/",
+        }
+        try:
+            async with (await cls._http_get(full_url, headers=headers)) as resp:
+                return await cls._response_to_str(resp, "gb2312")
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath("//div[@class='sort']//div[@class='list']/table")
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//td[@class='name']/h3/a/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            book_id = href.split("/")[-1].split(".")[0]
+            book_url = cls._abs_url(href)
+
+            cover_rel = cls._first_str(row.xpath(".//td[@class='cover']/a/img/@src"))
+            cover_url = cls._abs_url(cover_rel) if cover_rel else ""
+
+            title = cls._first_str(row.xpath(".//td[@class='name']/h3/a//text()"))
+
+            size_text = row.xpath("string(.//td[@class='size'])")
+            size_norm = size_text.replace("\u00a0", " ").replace(" ", " ").strip()
+            tokens = [t for t in size_norm.split() if t]
+
+            word_count = "-"
+            author = "-"
+            for tok in tokens:
+                if tok.startswith("大小:"):
+                    word_count = tok.split(":", 1)[1].strip()
+                elif tok.startswith("上传:"):
+                    author = tok.split(":", 1)[1].strip()
+
+            intro_text = row.xpath("string(.//td[@class='intro'])")
+            intro_norm = intro_text.replace("\u00a0", " ").replace(" ", " ")
+            update_date = "-"
+            for marker in ("更新:", "更新："):
+                if marker in intro_norm:
+                    tail = intro_norm.split(marker, 1)[1].strip()
+                    update_date = tail.split()[0] if tail else "-"
+                    break
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=cls.priority + idx,
+                )
+            )
+        return results

novel_downloader/core/searchers/b520.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.b520
+------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["biquge", "bqg", "b520"],
+)
+class BiqugeSearcher(BaseSearcher):
+    site_name = "biquge"
+    priority = 30
+    BASE_URL = "http://www.b520.cc/"
+    SEARCH_URL = "http://www.b520.cc/modules/article/search.php"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"searchkey": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath('//table[@class="grid"]//tr[position()>1]')
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            href = cls._first_str(row.xpath(".//td[1]/a[1]/@href"))
+            if not href:
+                continue
+
+            if limit is not None and idx >= limit:
+                break
+
+            book_id = href.strip("/").split("/")[-1]
+            book_url = cls._abs_url(href)
+
+            title = cls._first_str(row.xpath(".//td[1]/a[1]/text()"))
+
+            latest_chapter = cls._first_str(row.xpath(".//td[2]/a[1]/text()")) or "-"
+
+            author = cls._first_str(row.xpath(".//td[3]//text()"))
+            word_count = cls._first_str(row.xpath(".//td[4]//text()"))
+            update_date = cls._first_str(row.xpath(".//td[5]//text()"))
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url="",
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results