novel-downloader 1.5.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/__init__.py

@@ -3,35 +3,72 @@
 novel_downloader.core.parsers
 -----------------------------
 
-This package defines all site-specific parsing modules
-for the novel_downloader framework.
-
-Modules:
-- biquge (笔趣阁)
-- esjzone (ESJ Zone)
-- linovelib (哔哩轻小说)
-- qianbi (铅笔小说)
-- qidian (起点中文网)
-- sfacg (SF轻小说)
-- yamibo (百合会)
+Parser implementations for extracting book metadata and
+chapter content from various sources
 """
 
 __all__ = [
     "get_parser",
+    "AaatxtParser",
     "BiqugeParser",
+    "BiquyueduParser",
+    "DxmwxParser",
+    "EightnovelParser",
     "EsjzoneParser",
+    "GuidayeParser",
+    "HetushuParser",
+    "I25zwParser",
+    "Ixdzs8Parser",
+    "Jpxs123Parser",
+    "LewennParser",
     "LinovelibParser",
+    "PiaotiaParser",
+    "QbtrParser",
     "QianbiParser",
     "QidianParser",
+    "Quanben5Parser",
     "SfacgParser",
+    "ShencouParser",
+    "ShuhaigeParser",
+    "TongrenquanParser",
+    "TtkanParser",
+    "WanbengoParser",
+    "XiaoshuowuParser",
+    "XiguashuwuParser",
+    "Xs63bParser",
+    "XshbookParser",
     "YamiboParser",
+    "YibigeParser",
 ]
 
-from .biquge import BiqugeParser
+from .aaatxt import AaatxtParser
+from .b520 import BiqugeParser
+from .biquyuedu import BiquyueduParser
+from .dxmwx import DxmwxParser
+from .eightnovel import EightnovelParser
 from .esjzone import EsjzoneParser
+from .guidaye import GuidayeParser
+from .hetushu import HetushuParser
+from .i25zw import I25zwParser
+from .ixdzs8 import Ixdzs8Parser
+from .jpxs123 import Jpxs123Parser
+from .lewenn import LewennParser
 from .linovelib import LinovelibParser
+from .piaotia import PiaotiaParser
+from .qbtr import QbtrParser
 from .qianbi import QianbiParser
 from .qidian import QidianParser
+from .quanben5 import Quanben5Parser
 from .registry import get_parser
 from .sfacg import SfacgParser
+from .shencou import ShencouParser
+from .shuhaige import ShuhaigeParser
+from .tongrenquan import TongrenquanParser
+from .ttkan import TtkanParser
+from .wanbengo import WanbengoParser
+from .xiaoshuowu import XiaoshuowuParser
+from .xiguashuwu import XiguashuwuParser
+from .xs63b import Xs63bParser
+from .xshbook import XshbookParser
 from .yamibo import YamiboParser
+from .yibige import YibigeParser

novel_downloader/core/parsers/aaatxt.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.aaatxt
+------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["aaatxt"],
+)
+class AaatxtParser(BaseParser):
+    """
+    Parser for 3A电子书 book pages.
+    """
+
+    ADS: set[str] = {
+        "按键盘上方向键",
+        "未阅读完",
+        "加入书签",
+        "已便下次继续阅读",
+        "更多原创手机电子书",
+        "免费TXT小说下载",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath("//div[@class='xiazai']/h1/text()"))
+
+        author = self._first_str(tree.xpath("//span[@id='author']/a/text()"))
+
+        cover_url = self._first_str(
+            tree.xpath("//div[@id='txtbook']//div[@class='fm']//img/@src")
+        )
+
+        update_time = self._first_str(
+            tree.xpath("//div[@id='txtbook']//li[contains(text(), '上传日期')]/text()"),
+            replaces=[("上传日期:", "")],
+        )
+
+        genre = self._first_str(
+            tree.xpath("//div[@id='submenu']/h2/a[@class='lan']/text()")
+        )
+        tags = [genre] if genre else []
+
+        summary_el = tree.xpath("//div[@id='jj']//p")
+        summary = summary_el[0].text_content().strip() if summary_el else ""
+
+        download_url = self._first_str(
+            tree.xpath("//div[@id='down']//li[@class='bd']//a/@href")
+        )
+
+        # Chapters from the book_list
+        chapters: list[ChapterInfoDict] = []
+        for a in tree.xpath("//div[@id='ml']//ol/li/a"):
+            url = a.get("href", "").strip()
+            chapter_id = url.split("/")[-1].replace(".html", "")
+            title = a.text_content().strip()
+            chapters.append(
+                {
+                    "title": title,
+                    "url": url,
+                    "chapterId": chapter_id,
+                }
+            )
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {"download_url": download_url},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        raw_title = self._first_str(tree.xpath("//div[@id='content']//h1/text()"))
+        title = raw_title.split("-", 1)[-1].strip()
+
+        texts = []
+        for txt in tree.xpath("//div[@class='chapter']//text()"):
+            line = txt.strip()
+            # Skip empty/instruction/ad lines
+            if not line or self._is_ad_line(txt):
+                continue
+            texts.append(line)
+
+        content = "\n".join(texts)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "aaatxt"},
+        }

novel_downloader/core/parsers/b520.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.b520
+----------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["biquge", "bqg", "b520"],
+)
+class BiqugeParser(BaseParser):
+    """
+    Parser for 笔趣阁 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath('//div[@id="info"]/h1/text()'))
+
+        author = self._first_str(
+            tree.xpath('//div[@id="info"]/p[1]/text()'),
+            replaces=[("\xa0", ""), ("作者：", "")],
+        )
+
+        cover_url = self._first_str(tree.xpath('//div[@id="fmimg"]/img/@src'))
+
+        update_time = self._first_str(
+            tree.xpath('//div[@id="info"]/p[3]/text()'),
+            replaces=[("最后更新：", "")],
+        )
+
+        intro_elem = tree.xpath('//div[@id="intro"]')
+        summary = "".join(intro_elem[0].itertext()).strip() if intro_elem else ""
+
+        book_type = self._first_str(tree.xpath('//div[@class="con_top"]/a[2]/text()'))
+        tags = [book_type] if book_type else []
+
+        chapters: list[ChapterInfoDict] = [
+            {
+                "title": (a.text or "").strip(),
+                "url": (a.get("href") or "").strip(),
+                "chapterId": (a.get("href") or "").rsplit("/", 1)[-1].split(".", 1)[0],
+            }
+            for a in tree.xpath(
+                '//div[@id="list"]/dl/dt[contains(., "正文")]/following-sibling::dd/a'
+            )
+        ]
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath('//div[@class="bookname"]/h1/text()'))
+        if not title:
+            title = f"第 {chapter_id} 章"
+
+        content_elem = tree.xpath('//div[@id="content"]')
+        if not content_elem:
+            return None
+        paragraphs = [
+            "".join(p.itertext()).strip() for p in content_elem[0].xpath(".//p")
+        ]
+        if paragraphs and "www.shuhaige.net" in paragraphs[-1]:
+            paragraphs.pop()
+
+        content = "\n".join(paragraphs)
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "biquge"},
+        }

novel_downloader/core/parsers/base.py

@@ -3,22 +3,17 @@
 novel_downloader.core.parsers.base
 ----------------------------------
 
-This module defines the BaseParser abstract class, which implements the
-ParserProtocol interface and provides a structured foundation for
-site-specific parsers.
-
-BaseParser manages internal parser state and enforces
-a standard parsing interface for:
-- Book info pages (e.g. metadata, chapter list)
-- Chapter pages (e.g. textual content)
+Abstract base class providing common behavior for site-specific parsers.
 """
 
 import abc
+import re
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
 from novel_downloader.core.interfaces import ParserProtocol
-from novel_downloader.models import ChapterDict, ParserConfig
+from novel_downloader.models import BookInfoDict, ChapterDict, ParserConfig
 
 
 class BaseParser(ParserProtocol, abc.ABC):
@@ -32,6 +27,10 @@ class BaseParser(ParserProtocol, abc.ABC):
     Subclasses must implement actual parsing logic for specific sites.
     """
 
+    ADS: set[str] = set()
+
+    _SPACE_RE = re.compile(r"\s+")
+
     def __init__(
         self,
         config: ParserConfig,
@@ -44,15 +43,19 @@ class BaseParser(ParserProtocol, abc.ABC):
         self._config = config
         self._book_id: str | None = None
 
+        self._decode_font: bool = config.decode_font
+        self._use_truncation = config.use_truncation
         self._base_cache_dir = Path(config.cache_dir)
         self._cache_dir = self._base_cache_dir
 
+        self._ad_pattern = self._compile_ads_pattern()
+
     @abc.abstractmethod
     def parse_book_info(
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Parse and return a dictionary of book information from the raw HTML.
 
@@ -69,11 +72,11 @@ class BaseParser(ParserProtocol, abc.ABC):
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
-        Parse and return the text content of one chapter.
+        Parse chapter page and extract the content of one chapter.
 
         :param html_list: The HTML list of the chapter pages.
         :param chapter_id: Identifier of the chapter being parsed.
-        :return: The chapter's text.
+        :return: The chapter's data.
         """
         ...
 
@@ -104,3 +107,51 @@ class BaseParser(ParserProtocol, abc.ABC):
         book-related folders or states.
         """
         pass
+
+    def _compile_ads_pattern(self) -> re.Pattern[str] | None:
+        """
+        Compile a regex pattern from the ADS list, or return None if no ADS.
+        """
+        if not self.ADS:
+            return None
+
+        return re.compile("|".join(map(re.escape, self.ADS)))
+
+    def _is_ad_line(self, line: str) -> bool:
+        """
+        Check if a line contains any ad text.
+
+        :param line: Single text line.
+        :return: True if line matches ad pattern, else False.
+        """
+        return bool(self._ad_pattern and self._ad_pattern.search(line))
+
+    def _filter_ads(self, lines: Iterable[str]) -> list[str]:
+        """
+        Filter out lines containing any ad text defined in ADS.
+
+        :param lines: Iterable of text lines (e.g. chapter content).
+        :return: List of lines with ads removed.
+        """
+        if not self._ad_pattern:
+            return list(lines)
+        return [line for line in lines if not self._ad_pattern.search(line)]
+
+    @classmethod
+    def _norm_space(cls, s: str, c: str = " ") -> str:
+        """
+        collapse any run of whitespace (incl. newlines, full-width spaces)
+
+        :param s: Input string to normalize.
+        :param c: Replacement character to use for collapsed whitespace.
+        """
+        return cls._SPACE_RE.sub(c, s).strip()
+
+    @staticmethod
+    def _first_str(xs: list[str], replaces: list[tuple[str, str]] | None = None) -> str:
+        replaces = replaces or []
+        value: str = xs[0].strip() if xs else ""
+        for replace in replaces:
+            old, new = replace
+            value = value.replace(old, new)
+        return value

novel_downloader/core/parsers/biquyuedu.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.biquyuedu
+---------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import etree, html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["biquyuedu"],
+)
+class BiquyueduParser(BaseParser):
+    """
+    Parser for 精彩小说 book pages.
+    """
+
+    ADS: set[str] = {
+        "笔趣阁",
+        "请记住本书首发域名",
+        "www.biquyuedu.com",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # --- Metadata ---
+        book_name = self._first_str(tree.xpath("//div[@class='info']/h1/text()"))
+        author = self._first_str(
+            tree.xpath(
+                "//div[@class='info']//div[@class='small'][1]//span[1]//a/text()"
+            )
+        )
+        cover_url = self._first_str(
+            tree.xpath("//div[@class='info']//div[@class='cover']//img/@src")
+        )
+        update_time = self._first_str(
+            tree.xpath("//div[@class='info']//div[@class='small'][2]//span[1]/text()"),
+            replaces=[("更新时间：", "")],
+        )
+
+        crumbs = tree.xpath("//div[@class='path']//div[@class='p']/a/text()")
+        book_type = self._first_str(crumbs[1:2])
+        tags = [book_type] if book_type else []
+
+        intro_text = tree.xpath(
+            "string(//div[@class='info']//div[@class='intro'])"
+        ).strip()
+        summary = intro_text.replace("简介：", "", 1).split("作者：", 1)[0].strip()
+
+        # --- Chapters ---
+        chapters: list[ChapterInfoDict] = [
+            {
+                "title": (a.get("title") or a.text_content() or "").strip(),
+                "url": (a.get("href") or "").strip(),
+                "chapterId": (a.get("href") or "").rsplit("/", 1)[-1].split(".", 1)[0],
+            }
+            for a in tree.xpath(
+                "//div[@class='listmain']//dl/dd[preceding-sibling::dt[1][contains(text(),'全文')]]/a"
+            )
+        ]
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        # Extract chapter title via helper
+        title = self._first_str(tree.xpath("//div[@class='content']/h1/text()"))
+
+        # Find the main content container
+        content_nodes = tree.xpath("//div[@id='content']")
+        if not content_nodes:
+            return None
+        content_div = content_nodes[0]
+
+        etree.strip_elements(content_div, "script", with_tail=False)
+        raw_texts = content_div.xpath(".//text()[normalize-space()]")
+
+        # Clean & filter in one comprehension
+        paragraphs = [
+            txt.replace("\xa0", "").strip()
+            for txt in raw_texts
+            if not self._is_ad_line(txt)
+        ]
+
+        content = "\n".join(paragraphs)
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "biquyuedu"},
+        }