novel-downloader 1.5.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

novel_downloader/core/parsers/biquyuedu.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.biquyuedu
+---------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import etree, html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["biquyuedu"],
+)
+class BiquyueduParser(BaseParser):
+    """
+    Parser for 精彩小说 book pages.
+    """
+
+    ADS: set[str] = {
+        "笔趣阁",
+        "请记住本书首发域名",
+        "www.biquyuedu.com",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # --- Metadata ---
+        book_name = self._first_str(tree.xpath("//div[@class='info']/h1/text()"))
+        author = self._first_str(
+            tree.xpath(
+                "//div[@class='info']//div[@class='small'][1]//span[1]//a/text()"
+            )
+        )
+        cover_url = self._first_str(
+            tree.xpath("//div[@class='info']//div[@class='cover']//img/@src")
+        )
+        update_time = self._first_str(
+            tree.xpath("//div[@class='info']//div[@class='small'][2]//span[1]/text()"),
+            replaces=[("更新时间：", "")],
+        )
+
+        crumbs = tree.xpath("//div[@class='path']//div[@class='p']/a/text()")
+        book_type = self._first_str(crumbs[1:2])
+        tags = [book_type] if book_type else []
+
+        intro_text = tree.xpath(
+            "string(//div[@class='info']//div[@class='intro'])"
+        ).strip()
+        summary = intro_text.replace("简介：", "", 1).split("作者：", 1)[0].strip()
+
+        # --- Chapters ---
+        chapters: list[ChapterInfoDict] = [
+            {
+                "title": (a.get("title") or a.text_content() or "").strip(),
+                "url": (a.get("href") or "").strip(),
+                "chapterId": (a.get("href") or "").rsplit("/", 1)[-1].split(".", 1)[0],
+            }
+            for a in tree.xpath(
+                "//div[@class='listmain']//dl/dd[preceding-sibling::dt[1][contains(text(),'全文')]]/a"
+            )
+        ]
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        # Extract chapter title via helper
+        title = self._first_str(tree.xpath("//div[@class='content']/h1/text()"))
+
+        # Find the main content container
+        content_nodes = tree.xpath("//div[@id='content']")
+        if not content_nodes:
+            return None
+        content_div = content_nodes[0]
+
+        etree.strip_elements(content_div, "script", with_tail=False)
+        raw_texts = content_div.xpath(".//text()[normalize-space()]")
+
+        # Clean & filter in one comprehension
+        paragraphs = [
+            txt.replace("\xa0", "").strip()
+            for txt in raw_texts
+            if not self._is_ad_line(txt)
+        ]
+
+        content = "\n".join(paragraphs)
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "biquyuedu"},
+        }

novel_downloader/core/parsers/dxmwx.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.dxmwx
+-----------------------------------
+
+"""
+
+import re
+from datetime import datetime
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["dxmwx"],
+)
+class DxmwxParser(BaseParser):
+    """
+    Parser for 大熊猫文学网 book pages.
+    """
+
+    _RE_DATE = re.compile(r"\d{4}-\d{2}-\d{2}")
+    _RE_SPACES = re.compile(r"[ \t\u3000]+")
+    _RE_NEWLINES = re.compile(r"\n{2,}")
+    _RE_TITLE_WS = re.compile(r"\s+")
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if len(html_list) < 2:
+            return None
+
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
+
+        book_name = self._first_str(
+            info_tree.xpath("//span[contains(@style,'font-size: 24px')]/text()")
+        )
+        author = self._first_str(
+            info_tree.xpath(
+                "//div[contains(@style,'height: 28px') and contains(., '著')]//a/text()"
+            )
+        )
+        tags = [
+            t.strip()
+            for t in info_tree.xpath("//span[@class='typebut']//a/text()")
+            if t.strip()
+        ]
+        cover_url = "https://www.dxmwx.org" + self._first_str(
+            info_tree.xpath("//img[@class='imgwidth']/@src")
+        )
+
+        raw_update = self._first_str(
+            info_tree.xpath(
+                "normalize-space(string(//span[starts-with(normalize-space(.), '更新时间：')]))"  # noqa: E501
+            )
+        )
+        raw_update = raw_update.replace("更新时间：", "").strip()
+        update_time = self._normalize_update_date(raw_update)
+
+        nodes = info_tree.xpath(
+            "//div[contains(@style,'min-height') and "
+            "contains(@style,'padding-left') and contains(@style,'padding-right')][1]"
+        )
+        summary = ""
+        if nodes:
+            texts = [
+                t.replace("\xa0", " ").strip() for t in nodes[0].xpath(".//text()")
+            ]
+            lines = [t for t in texts if t]
+            summary = "\n".join(lines)
+            summary = re.sub(r"^\s*[:：]\s*", "", summary)
+            summary = self._clean_spaces(summary)
+
+        chapters: list[ChapterInfoDict] = []
+        for a in catalog_tree.xpath(
+            "//div[contains(@style,'height:40px') and contains(@style,'border-bottom')]//a"  # noqa: E501
+        ):
+            href = a.get("href") or ""
+            title = (a.text_content() or "").strip()
+            if not href or not title:
+                continue
+            # "/read/57215_50197663.html" -> "50197663"
+            chap_id = href.split("read/", 1)[-1].split(".html", 1)[0].split("_")[-1]
+            chapters.append({"title": title, "url": href, "chapterId": chap_id})
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath("//h1[@id='ChapterTitle']/text()"))
+        title = self._RE_TITLE_WS.sub(" ", title).strip()
+        if not title:
+            title = f"第 {chapter_id} 章"
+
+        paragraphs: list[str] = []
+        for p in tree.xpath("//div[@id='Lab_Contents']//p"):
+            text = self._clean_spaces(p.text_content())
+            if not text:
+                continue
+            if "点这里听书" in text or "大熊猫文学" in text:
+                continue
+            paragraphs.append(text)
+
+        content = "\n".join(paragraphs).strip()
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "dxmwx"},
+        }
+
+    @classmethod
+    def _clean_spaces(cls, s: str) -> str:
+        s = s.replace("\xa0", " ")
+        s = cls._RE_SPACES.sub(" ", s)
+        s = cls._RE_NEWLINES.sub("\n", s)
+        return s.strip()
+
+    @classmethod
+    def _normalize_update_date(cls, raw: str) -> str:
+        """Return a YYYY-MM-DD string."""
+        if not raw:
+            return datetime.now().strftime("%Y-%m-%d")
+        m = cls._RE_DATE.search(raw)
+        if m:
+            return m.group(0)
+        return datetime.now().strftime("%Y-%m-%d")

novel_downloader/core/parsers/eightnovel.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.eightnovel
+----------------------------------------
+
+"""
+
+import re
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["eightnovel", "8novel"],
+)
+class EightnovelParser(BaseParser):
+    """
+    Parser for 无限轻小说 book pages.
+    """
+
+    BASE_URL = "https://www.8novel.com"
+    _SPLIT_STR_PATTERN = re.compile(
+        r'["\']([^"\']+)["\']\s*\.split\s*\(\s*["\']\s*,\s*["\']\s*\)', re.DOTALL
+    )
+    _RE_AUTHOR = re.compile(r"作者[:：]?\s*")
+    _RE_UPDATE = re.compile(r"更新[:：]?\s*")
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # --- Basic metadata ---
+        book_name = self._first_str(tree.xpath("//li[contains(@class,'h2')]/text()"))
+
+        author_raw = self._first_str(
+            tree.xpath("//span[contains(@class,'item-info-author')]/text()")
+        )
+        author = self._RE_AUTHOR.sub("", author_raw)
+
+        cover_url = self.BASE_URL + self._first_str(
+            tree.xpath("//div[contains(@class,'item-cover')]//img/@src")
+        )
+
+        update_raw = self._first_str(
+            tree.xpath("//span[contains(@class,'item-info-date')]/text()")
+        )
+        update_time = self._RE_UPDATE.sub("", update_raw)
+
+        counts = tree.xpath(
+            "//li[@class='small text-gray']//span[contains(@class,'item-info-num')]/text()"  # noqa: E501
+        )
+        word_count = counts[1].strip() + "萬字" if len(counts) >= 2 else ""
+
+        tags = tree.xpath("//meta[@property='og:novel:category']/@content")
+
+        # --- Summary ---
+        summary_nodes = tree.xpath(
+            "//li[contains(@class,'full_text') and contains(@class,'mt-2')]"
+        )
+        if summary_nodes:
+            texts = [t.strip() for t in summary_nodes[0].itertext()]
+            summary = "\n".join(line for line in texts if line)
+        else:
+            summary = ""
+
+        # --- Chapters / Volumes ---
+        volumes: list[VolumeInfoDict] = []
+        for vol_div in tree.xpath("//div[contains(@class,'folder') and @pid]"):
+            # Volume title
+            h3 = vol_div.xpath(".//div[contains(@class,'vol-title')]//h3")
+            vol_name = (
+                h3[0].text_content().split("/")[0].strip() if h3 else "Unnamed Volume"
+            )
+
+            # Chapters
+            chapters: list[ChapterInfoDict] = []
+            for a in vol_div.xpath(
+                ".//a[contains(@class,'episode_li') and contains(@class,'d-block')]"
+            ):
+                title = (a.text_content() or "").strip()
+                href = a.get("href") or ""
+                if not href or not title:
+                    continue
+                url = href if href.startswith("http") else self.BASE_URL + href
+                chapter_id = href.split("?")[-1]  # "/read/3355/?270015" -> "270015"
+                chapters.append({"title": title, "url": url, "chapterId": chapter_id})
+
+            volumes.append({"volume_name": vol_name, "chapters": chapters})
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "word_count": word_count,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if len(html_list) < 2:
+            return None
+
+        try:
+            id_title_map = self._build_id_title_map(html_list[0])
+            title = id_title_map.get(chapter_id) or ""
+        except Exception:
+            title = ""
+
+        wrapper = html.fromstring(f"<div>{html_list[1]}</div>")
+
+        segments: list[str] = []
+
+        self._append_segment(segments, wrapper.text)
+
+        for node in wrapper:
+            tag = node.tag.lower() if isinstance(node.tag, str) else ""
+
+            # A picture‑gallery block
+            if tag == "div" and "content-pics" in (node.get("class") or ""):
+                for img in node.xpath(".//img"):
+                    src = img.get("src")
+                    full = src if not src.startswith("/") else self.BASE_URL + src
+                    segments.append(f'<img src="{full}" />')
+                self._append_segment(segments, node.tail)
+
+            # Standalone <img>
+            elif tag == "img":
+                src = node.get("src")
+                if not src:
+                    continue
+                full = src if not src.startswith("/") else self.BASE_URL + src
+                segments.append(f'<img src="{full}" />')
+                self._append_segment(segments, node.tail)
+
+            # Line break -> text in .tail is next paragraph
+            elif tag == "br":
+                self._append_segment(segments, node.tail)
+
+            # Any other element -> get its text content
+            else:
+                self._append_segment(segments, node.text_content())
+                self._append_segment(segments, node.tail)
+
+        # Remove final ad line if present
+        if segments and segments[-1] and segments[-1][0] in ("8", "⑧", "⒏"):
+            segments.pop()
+
+        content = "\n".join(segments).strip()
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "eightnovel"},
+        }
+
+    @staticmethod
+    def _append_segment(segments: list[str], text: str | None) -> None:
+        """
+        Strip, filter out the '8novel' ad, and append non-empty text to segments.
+        """
+        if not text:
+            return
+        cleaned = text.strip()
+        if cleaned:
+            segments.append(cleaned)
+
+    @classmethod
+    def _build_id_title_map(cls, html_str: str) -> dict[str, str]:
+        """
+        Extracts two comma-split lists from html_str:
+          * A numeric list of IDs (one element longer)
+          * A list of titles
+        """
+        id_list = None
+        title_list = None
+
+        for content in cls._SPLIT_STR_PATTERN.findall(html_str):
+            items = [s.strip() for s in content.split(",")]
+            if items == [""]:
+                # skip bids=""
+                continue
+            if all(item.isdigit() for item in items):
+                id_list = items
+            else:
+                title_list = items
+
+            if id_list and title_list:
+                break
+
+        if not id_list or not title_list:
+            raise ValueError("Could not locate both ID and title lists")
+        if len(id_list) != len(title_list) + 1:
+            raise ValueError(
+                "ID list must be exactly one element longer than title list"
+            )
+
+        return dict(zip(id_list[:-1], title_list, strict=False))