novel-downloader 1.5.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/wanbengo.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.wanbengo
+--------------------------------------
+
+"""
+
+import re
+from datetime import datetime
+from html import unescape
+from typing import Any
+from urllib.parse import urljoin
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["wanbengo"],
+)
+class WanbengoParser(BaseParser):
+    """
+    Parser for 完本神站 book pages.
+    """
+
+    BASE = "https://www.wanbengo.com"
+
+    # XPaths for the book info page
+    X_BOOK_NAME = "//div[@class='detailTopMid']//h1/text()"
+    X_AUTHOR = "//div[@class='detailTopMid']//div[@class='writer']//a/text()"
+    X_COVER = "//div[@class='detailTopLeft']//img/@src"
+    X_STATUS = "//div[@class='detailTopLeft']//span[contains(@class,'end')]/text()"
+    X_WORDS = "//div[@class='detailTopMid']//table//tr[td/span[contains(text(),'字数')]]/td[last()]/text()"  # noqa: E501
+    X_SUMMARY = "//div[@class='detailTopMid']//table//tr[td/span[contains(text(),'简介')]]/td[last()]//text()"  # noqa: E501
+    X_TAG = "//div[@class='route']/a[2]//text()"
+    X_UPDATE_TXT = "//div[@class='chapterTitle']//span//text()"
+    X_CHAPTERS = "//div[@class='chapter']//ul//li/a"
+
+    # XPaths for the chapter page
+    X_CHAP_TITLE = "//div[contains(@class,'readerTitle')]//h2/text()"
+    _CHAP_SPLIT_RE = re.compile(r"(?:</p\s*>|<p\b[^>]*>|<br\s*/?>)", re.I)
+    _CHAP_READERCON_RE = re.compile(
+        r'<div[^>]*class=(?:"[^"]*readerCon[^"]*"|\'[^\']*readerCon[^\']*\')[^>]*>(.*?)</div>',
+        re.I | re.S,
+    )
+    _TAGS_RE = re.compile(r"<[^>]+>")
+    _SCRUB_RUNS_RE = re.compile(r"[_?]{2,}")
+    _SCRUB_TAIL_RE = re.compile(r"\s*（未完待续.*?$")
+
+    # fmt: off
+    ADS = {
+        "完本神站", "本站网址", "报错", "键盘", "客户端", "收藏", "书架",
+        "猜你喜欢", "上一章", "下一章", "章节目录", "LastRead", "贴吧",
+        "倾心打造", "全文无错", "分享本站", "点此章节报错", "温馨提示", "域名",
+        "wanbentxt.com", "wanbengo.com",
+    }
+    # fmt: on
+    _PUNCT_ONLY = re.compile(
+        r"^[\s\W_·—\-･。，、；;：:！!？?\(\)（）【】《》“”\"'…·]+$"
+    )  # noqa: E501
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath(self.X_BOOK_NAME))
+        author = self._first_str(tree.xpath(self.X_AUTHOR))
+        cover_url = self._first_str(tree.xpath(self.X_COVER))
+        serial_status = (
+            self._norm_space(self._first_str(tree.xpath(self.X_STATUS))) or "连载中"
+        )
+        word_count = self._norm_space("".join(tree.xpath(self.X_WORDS)))
+        summary = self._norm_space("".join(tree.xpath(self.X_SUMMARY)))
+
+        book_type = self._norm_space("".join(tree.xpath(self.X_TAG)))
+        tags = [book_type] if book_type else []
+
+        update_time = self._extract_update_date(tree.xpath(self.X_UPDATE_TXT))
+
+        chapters: list[ChapterInfoDict] = []
+        for a in tree.xpath(self.X_CHAPTERS):
+            title = self._norm_space("".join(a.xpath(".//text()")))
+            href = a.get("href") or ""
+            url = urljoin(self.BASE, href)
+            # "/129/103950.html" -> "103950"
+            cid = url.rstrip(".html").split("/")[-1]
+            chapters.append({"title": title, "url": url, "chapterId": cid})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "word_count": word_count,
+            "summary": summary,
+            "tags": tags,
+            "volumes": volumes,
+            "serial_status": serial_status,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        inner = self._CHAP_READERCON_RE.search(html_list[0])
+        if not inner:
+            return None
+
+        tree = html.fromstring(html_list[0])
+        title = self._first_str(tree.xpath(self.X_CHAP_TITLE))
+
+        parts = self._CHAP_SPLIT_RE.split(inner.group(1))
+        lines: list[str] = []
+        for part in parts:
+            if not part:
+                continue
+            s = self._TAGS_RE.sub("", part)
+            s = unescape(s).replace("\xa0", " ")
+            if self._is_noise_line(s):
+                continue
+            s = self._norm_space(self._scrub_ascii_gibberish(s.strip()))
+            if s:
+                lines.append(s)
+
+        content = "\n".join(lines)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "wanbengo"},
+        }
+
+    @staticmethod
+    def _extract_update_date(texts: list[str]) -> str:
+        """
+        Find a YYYY-MM-DD anywhere in the provided text nodes.
+
+        If none found, return today's date.
+        """
+        joined = " ".join(t for t in texts if t)
+        m = re.search(r"\b(\d{4}-\d{2}-\d{2})\b", joined)
+        if m:
+            return m.group(1)
+        return datetime.now().strftime("%Y-%m-%d")
+
+    def _is_noise_line(self, s: str) -> bool:
+        """Heuristic to drop obvious ad/footer/noise lines."""
+        if not s.strip():
+            return True
+        if self._is_ad_line(s):
+            return True
+        if self._PUNCT_ONLY.match(s):
+            return True
+        return False
+
+    @classmethod
+    def _scrub_ascii_gibberish(cls, s: str) -> str:
+        """
+        Remove common injected ASCII junk like long runs of '?' or '_'
+        while keeping normal text intact.
+        """
+        s = s.replace("()?()", "").replace("[(．)]", "")
+        s = s.replace("．", ".")
+        s = cls._SCRUB_RUNS_RE.sub("", s)  # drop runs like ???? or ____
+        s = cls._SCRUB_TAIL_RE.sub("", s)
+        return s.strip()

novel_downloader/core/parsers/xiaoshuowu.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.xiaoshuowu
+----------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["xiaoshuowu", "xiaoshuoge"],
+)
+class XiaoshuowuParser(BaseParser):
+    """
+    Parser for 小说屋 (xiaoshuoge.info).
+    """
+
+    AD_STR: str = "小说屋 www.xiaoshuoge.info"
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if len(html_list) < 2:
+            return None
+
+        # Parse trees
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
+
+        book_name = self._first_str(
+            info_tree.xpath('//meta[@property="og:novel:book_name"]/@content')
+        )
+        author = self._first_str(
+            info_tree.xpath('//meta[@property="og:novel:author"]/@content')
+        )
+
+        # Category -> tags
+        cat_val = self._first_str(
+            info_tree.xpath('//meta[@property="og:novel:category"]/@content')
+        )
+        tags = [cat_val] if cat_val else []
+
+        word_count = self._first_str(
+            info_tree.xpath(
+                '//table[@class="hide"]//td[contains(text(),"全文字数")]/text()'
+            ),
+            replaces=[("全文字数：", "")],
+        )
+        update_time = self._first_str(
+            info_tree.xpath(
+                '//table[@class="hide"]//td[contains(text(),"最后更新")]/text()'
+            ),
+            replaces=[("最后更新：", "")],
+        )
+        serial_status = self._first_str(
+            info_tree.xpath(
+                '//table[@class="hide"]//td[contains(text(),"连载状态")]/text()'
+            ),
+            replaces=[("连载状态：", "")],
+        )
+
+        cover_url = self._first_str(
+            info_tree.xpath('//meta[@property="og:image"]/@content')
+        )
+
+        # Summary
+        summary_div = info_tree.xpath('//div[@class="tabvalue"][1]//div')
+        summary: str = summary_div[0].text_content().strip() if summary_div else ""
+
+        # Chapters (single volume)
+        chapters: list[ChapterInfoDict] = []
+        chapter_links = catalog_tree.xpath(
+            '//ul[contains(@class,"chapters")]//li[contains(@class,"chapter")]/a'
+        )
+        for a in chapter_links:
+            url = a.get("href", "").strip()
+            title = a.text_content().strip()
+            # chapterId is the numeric filename before ".html"
+            chapter_id = url.rsplit("/", 1)[-1].split(".")[0]
+            chapters.append({"title": title, "url": url, "chapterId": chapter_id})
+
+        # Single volume
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "word_count": word_count,
+            "serial_status": serial_status,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        doc = html.fromstring(html_list[0])
+        # main container
+        content_divs = doc.xpath('//div[@id="acontent"]')
+        if not content_divs:
+            return None
+        container = content_divs[0]
+
+        # Get the <h1> title
+        title_elem = container.find("h1")
+        title = title_elem.text_content().strip() if title_elem is not None else ""
+
+        paras: list[str] = []
+        started = False
+        for node in container.xpath("./*"):
+            # anchor: first <div id="content_tip">
+            if node.tag == "div" and node.get("id") == "content_tip":
+                raw = node.tail or ""
+                # drop any "(小说屋 ...)" prefix before the real text
+                if ")" in raw:
+                    raw = raw.split(")", 1)[1]
+                first_line = raw.lstrip("\ufeff").strip()
+                if first_line:
+                    paras.append(first_line)
+                started = True
+                continue
+
+            if not started:
+                continue
+
+            # stop collecting once we hit any div
+            cls_name = node.get("class") or ""
+            if node.tag == "div" and any(
+                k in cls_name for k in ("tishi", "footlink", "fullbar")
+            ):
+                break
+
+            # grab each <br/> tail as a paragraph
+            if node.tag == "br":
+                line = (node.tail or "").strip()
+                if not line or self.AD_STR in line:
+                    continue
+                paras.append(line)
+
+        if not paras:
+            return None
+        content = "\n".join(paras)
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "xiaoshuowu"},
+        }