novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/interfaces/downloader.py

@@ -3,10 +3,10 @@
 novel_downloader.core.interfaces.downloader
 -------------------------------------------
 
-This module defines the DownloaderProtocol, a structural interface
-that outlines the expected behavior of any downloader class.
+Protocol defining the interface for asynchronous book downloaders.
 """
 
+import asyncio
 from collections.abc import Awaitable, Callable
 from typing import Any, Protocol, runtime_checkable
 
@@ -27,6 +27,7 @@ class DownloaderProtocol(Protocol):
         book: BookConfig,
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -35,6 +36,7 @@ class DownloaderProtocol(Protocol):
         :param book: BookConfig with at least 'book_id'.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         ...
 
@@ -43,6 +45,7 @@ class DownloaderProtocol(Protocol):
         books: list[BookConfig],
         *,
         progress_hook: Callable[[int, int], Awaitable[None]] | None = None,
+        cancel_event: asyncio.Event | None = None,
         **kwargs: Any,
     ) -> None:
         """
@@ -51,5 +54,6 @@ class DownloaderProtocol(Protocol):
         :param books: List of BookConfig entries.
         :param progress_hook: Optional async callback after each chapter.
                                 args: completed_count, total_count.
+        :param cancel_event: Optional asyncio.Event to allow cancellation.
         """
         ...

novel_downloader/core/interfaces/exporter.py

@@ -3,10 +3,10 @@
 novel_downloader.core.interfaces.exporter
 -----------------------------------------
 
-Defines the ExporterProtocol interface for persisting completed books in
-TXT, EPUB, Markdown, and PDF formats.
+Protocol defining the interface for exporting books to text, EPUB, and other formats.
 """
 
+from pathlib import Path
 from typing import Protocol, runtime_checkable
 
 
@@ -18,7 +18,7 @@ class ExporterProtocol(Protocol):
     It may also optionally implement an EPUB (or other format) exporter.
     """
 
-    def export(self, book_id: str) -> None:
+    def export(self, book_id: str) -> dict[str, Path]:
         """
         Export the book in the formats specified in config.
         If a method is not implemented or fails, log the error and continue.
@@ -27,7 +27,7 @@ class ExporterProtocol(Protocol):
         """
         ...
 
-    def export_as_txt(self, book_id: str) -> None:
+    def export_as_txt(self, book_id: str) -> Path | None:
         """
         Persist the assembled book as a .txt file.
 
@@ -35,7 +35,7 @@ class ExporterProtocol(Protocol):
         """
         ...
 
-    def export_as_epub(self, book_id: str) -> None:
+    def export_as_epub(self, book_id: str) -> Path | None:
         """
         Optional: Persist the assembled book as an .epub file.
 
@@ -43,7 +43,7 @@ class ExporterProtocol(Protocol):
         """
         ...
 
-    def export_as_md(self, book_id: str) -> None:
+    def export_as_md(self, book_id: str) -> Path | None:
         """
         Optional: Persist the assembled book as a Markdown (.md) file.
 
@@ -51,7 +51,7 @@ class ExporterProtocol(Protocol):
         """
         ...
 
-    def export_as_pdf(self, book_id: str) -> None:
+    def export_as_pdf(self, book_id: str) -> Path | None:
         """
         Optional: Persist the assembled book as a PDF file.
 

novel_downloader/core/interfaces/fetcher.py

@@ -3,8 +3,7 @@
 novel_downloader.core.interfaces.fetcher
 ----------------------------------------
 
-Defines the Async FetcherProtocol interface for fetching raw HTML or JSON
-for book info pages, individual chapters, managing request lifecycle
+Protocol defining the interface for asynchronous fetching, login, and session management
 """
 
 import types
@@ -32,6 +31,7 @@ class FetcherProtocol(Protocol):
     ) -> bool:
         """
         Attempt to log in asynchronously.
+
         :returns: True if login succeeded.
         """
         ...
@@ -45,7 +45,7 @@ class FetcherProtocol(Protocol):
         Fetch the raw HTML (or JSON) of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         ...
 
@@ -60,7 +60,7 @@ class FetcherProtocol(Protocol):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as string.
+        :return: The page content as string list.
         """
         ...
 
@@ -126,19 +126,6 @@ class FetcherProtocol(Protocol):
         """
         ...
 
-    async def set_interactive_mode(self, enable: bool) -> bool:
-        """
-        Enable or disable interactive mode for manual login.
-
-        :param enable: True to enable, False to disable interactive mode.
-        :return: True if operation or login check succeeded, False otherwise.
-        """
-        ...
-
-    @property
-    def requester_type(self) -> str:
-        ...
-
     @property
     def is_logged_in(self) -> bool:
         """

novel_downloader/core/interfaces/parser.py

@@ -3,13 +3,12 @@
 novel_downloader.core.interfaces.parser
 ---------------------------------------
 
-Defines the ParserProtocol interface for extracting book metadata,
-parsing individual chapter content, and setting parser context via book_id.
+Protocol defining the interface for parsing book metadata and chapter content.
 """
 
 from typing import Any, Protocol, runtime_checkable
 
-from novel_downloader.models import ChapterDict
+from novel_downloader.models import BookInfoDict, ChapterDict
 
 
 @runtime_checkable
@@ -24,7 +23,7 @@ class ParserProtocol(Protocol):
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Parse and return a dictionary of book information from the raw HTML.
 
@@ -40,10 +39,10 @@ class ParserProtocol(Protocol):
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
-        Parse and return the text content of one chapter.
+        Parse chapter page and extract the content of one chapter.
 
         :param html_list: The HTML list of the chapter pages.
         :param chapter_id: Identifier of the chapter being parsed.
-        :return: The chapter's text.
+        :return: The chapter's data.
         """
         ...

novel_downloader/core/interfaces/searcher.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.interfaces.searcher
+-----------------------------------------
+
+Protocol defining the interface for site search implementations.
+"""
+
+from typing import Protocol
+
+import aiohttp
+
+from novel_downloader.models import SearchResult
+
+
+class SearcherProtocol(Protocol):
+    site_name: str
+
+    @classmethod
+    def configure(cls, session: aiohttp.ClientSession) -> None:
+        """Configure the shared session"""
+        ...
+
+    @classmethod
+    async def search(cls, keyword: str, limit: int | None = None) -> list[SearchResult]:
+        ...

novel_downloader/core/parsers/__init__.py

@@ -3,36 +3,72 @@
 novel_downloader.core.parsers
 -----------------------------
 
-This package defines all site-specific parsing modules
-for the novel_downloader framework.
-
-Modules:
-- biquge (笔趣阁)
-- esjzone (ESJ Zone)
-- linovelib (哔哩轻小说)
-- qianbi (铅笔小说)
-- qidian (起点中文网)
-- sfacg (SF轻小说)
-- yamibo (百合会)
-- common (通用架构)
+Parser implementations for extracting book metadata and
+chapter content from various sources
 """
 
-from .biquge import BiqugeParser
-from .common import CommonParser
-from .esjzone import EsjzoneParser
-from .linovelib import LinovelibParser
-from .qianbi import QianbiParser
-from .qidian import QidianParser
-from .sfacg import SfacgParser
-from .yamibo import YamiboParser
-
 __all__ = [
+    "get_parser",
+    "AaatxtParser",
     "BiqugeParser",
-    "CommonParser",
+    "BiquyueduParser",
+    "DxmwxParser",
+    "EightnovelParser",
     "EsjzoneParser",
+    "GuidayeParser",
+    "HetushuParser",
+    "I25zwParser",
+    "Ixdzs8Parser",
+    "Jpxs123Parser",
+    "LewennParser",
     "LinovelibParser",
+    "PiaotiaParser",
+    "QbtrParser",
     "QianbiParser",
     "QidianParser",
+    "Quanben5Parser",
     "SfacgParser",
+    "ShencouParser",
+    "ShuhaigeParser",
+    "TongrenquanParser",
+    "TtkanParser",
+    "WanbengoParser",
+    "XiaoshuowuParser",
+    "XiguashuwuParser",
+    "Xs63bParser",
+    "XshbookParser",
     "YamiboParser",
+    "YibigeParser",
 ]
+
+from .aaatxt import AaatxtParser
+from .b520 import BiqugeParser
+from .biquyuedu import BiquyueduParser
+from .dxmwx import DxmwxParser
+from .eightnovel import EightnovelParser
+from .esjzone import EsjzoneParser
+from .guidaye import GuidayeParser
+from .hetushu import HetushuParser
+from .i25zw import I25zwParser
+from .ixdzs8 import Ixdzs8Parser
+from .jpxs123 import Jpxs123Parser
+from .lewenn import LewennParser
+from .linovelib import LinovelibParser
+from .piaotia import PiaotiaParser
+from .qbtr import QbtrParser
+from .qianbi import QianbiParser
+from .qidian import QidianParser
+from .quanben5 import Quanben5Parser
+from .registry import get_parser
+from .sfacg import SfacgParser
+from .shencou import ShencouParser
+from .shuhaige import ShuhaigeParser
+from .tongrenquan import TongrenquanParser
+from .ttkan import TtkanParser
+from .wanbengo import WanbengoParser
+from .xiaoshuowu import XiaoshuowuParser
+from .xiguashuwu import XiguashuwuParser
+from .xs63b import Xs63bParser
+from .xshbook import XshbookParser
+from .yamibo import YamiboParser
+from .yibige import YibigeParser

novel_downloader/core/parsers/aaatxt.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.aaatxt
+------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["aaatxt"],
+)
+class AaatxtParser(BaseParser):
+    """
+    Parser for 3A电子书 book pages.
+    """
+
+    ADS: set[str] = {
+        "按键盘上方向键",
+        "未阅读完",
+        "加入书签",
+        "已便下次继续阅读",
+        "更多原创手机电子书",
+        "免费TXT小说下载",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath("//div[@class='xiazai']/h1/text()"))
+
+        author = self._first_str(tree.xpath("//span[@id='author']/a/text()"))
+
+        cover_url = self._first_str(
+            tree.xpath("//div[@id='txtbook']//div[@class='fm']//img/@src")
+        )
+
+        update_time = self._first_str(
+            tree.xpath("//div[@id='txtbook']//li[contains(text(), '上传日期')]/text()"),
+            replaces=[("上传日期:", "")],
+        )
+
+        genre = self._first_str(
+            tree.xpath("//div[@id='submenu']/h2/a[@class='lan']/text()")
+        )
+        tags = [genre] if genre else []
+
+        summary_el = tree.xpath("//div[@id='jj']//p")
+        summary = summary_el[0].text_content().strip() if summary_el else ""
+
+        download_url = self._first_str(
+            tree.xpath("//div[@id='down']//li[@class='bd']//a/@href")
+        )
+
+        # Chapters from the book_list
+        chapters: list[ChapterInfoDict] = []
+        for a in tree.xpath("//div[@id='ml']//ol/li/a"):
+            url = a.get("href", "").strip()
+            chapter_id = url.split("/")[-1].replace(".html", "")
+            title = a.text_content().strip()
+            chapters.append(
+                {
+                    "title": title,
+                    "url": url,
+                    "chapterId": chapter_id,
+                }
+            )
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {"download_url": download_url},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        raw_title = self._first_str(tree.xpath("//div[@id='content']//h1/text()"))
+        title = raw_title.split("-", 1)[-1].strip()
+
+        texts = []
+        for txt in tree.xpath("//div[@class='chapter']//text()"):
+            line = txt.strip()
+            # Skip empty/instruction/ad lines
+            if not line or self._is_ad_line(txt):
+                continue
+            texts.append(line)
+
+        content = "\n".join(texts)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "aaatxt"},
+        }

novel_downloader/core/parsers/b520.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.b520
+----------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["biquge", "bqg", "b520"],
+)
+class BiqugeParser(BaseParser):
+    """
+    Parser for 笔趣阁 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        book_name = self._first_str(tree.xpath('//div[@id="info"]/h1/text()'))
+
+        author = self._first_str(
+            tree.xpath('//div[@id="info"]/p[1]/text()'),
+            replaces=[("\xa0", ""), ("作者：", "")],
+        )
+
+        cover_url = self._first_str(tree.xpath('//div[@id="fmimg"]/img/@src'))
+
+        update_time = self._first_str(
+            tree.xpath('//div[@id="info"]/p[3]/text()'),
+            replaces=[("最后更新：", "")],
+        )
+
+        intro_elem = tree.xpath('//div[@id="intro"]')
+        summary = "".join(intro_elem[0].itertext()).strip() if intro_elem else ""
+
+        book_type = self._first_str(tree.xpath('//div[@class="con_top"]/a[2]/text()'))
+        tags = [book_type] if book_type else []
+
+        chapters: list[ChapterInfoDict] = [
+            {
+                "title": (a.text or "").strip(),
+                "url": (a.get("href") or "").strip(),
+                "chapterId": (a.get("href") or "").rsplit("/", 1)[-1].split(".", 1)[0],
+            }
+            for a in tree.xpath(
+                '//div[@id="list"]/dl/dt[contains(., "正文")]/following-sibling::dd/a'
+            )
+        ]
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+        tree = html.fromstring(html_list[0])
+
+        title = self._first_str(tree.xpath('//div[@class="bookname"]/h1/text()'))
+        if not title:
+            title = f"第 {chapter_id} 章"
+
+        content_elem = tree.xpath('//div[@id="content"]')
+        if not content_elem:
+            return None
+        paragraphs = [
+            "".join(p.itertext()).strip() for p in content_elem[0].xpath(".//p")
+        ]
+        if paragraphs and "www.shuhaige.net" in paragraphs[-1]:
+            paragraphs.pop()
+
+        content = "\n".join(paragraphs)
+        if not content.strip():
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "biquge"},
+        }

novel_downloader/core/parsers/base.py

@@ -3,22 +3,17 @@
 novel_downloader.core.parsers.base
 ----------------------------------
 
-This module defines the BaseParser abstract class, which implements the
-ParserProtocol interface and provides a structured foundation for
-site-specific parsers.
-
-BaseParser manages internal parser state and enforces
-a standard parsing interface for:
-- Book info pages (e.g. metadata, chapter list)
-- Chapter pages (e.g. textual content)
+Abstract base class providing common behavior for site-specific parsers.
 """
 
 import abc
+import re
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
 from novel_downloader.core.interfaces import ParserProtocol
-from novel_downloader.models import ChapterDict, ParserConfig
+from novel_downloader.models import BookInfoDict, ChapterDict, ParserConfig
 
 
 class BaseParser(ParserProtocol, abc.ABC):
@@ -32,6 +27,10 @@ class BaseParser(ParserProtocol, abc.ABC):
     Subclasses must implement actual parsing logic for specific sites.
     """
 
+    ADS: set[str] = set()
+
+    _SPACE_RE = re.compile(r"\s+")
+
     def __init__(
         self,
         config: ParserConfig,
@@ -44,15 +43,19 @@ class BaseParser(ParserProtocol, abc.ABC):
         self._config = config
         self._book_id: str | None = None
 
+        self._decode_font: bool = config.decode_font
+        self._use_truncation = config.use_truncation
         self._base_cache_dir = Path(config.cache_dir)
         self._cache_dir = self._base_cache_dir
 
+        self._ad_pattern = self._compile_ads_pattern()
+
     @abc.abstractmethod
     def parse_book_info(
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Parse and return a dictionary of book information from the raw HTML.
 
@@ -69,11 +72,11 @@ class BaseParser(ParserProtocol, abc.ABC):
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
-        Parse and return the text content of one chapter.
+        Parse chapter page and extract the content of one chapter.
 
         :param html_list: The HTML list of the chapter pages.
         :param chapter_id: Identifier of the chapter being parsed.
-        :return: The chapter's text.
+        :return: The chapter's data.
         """
         ...
 
@@ -104,3 +107,51 @@ class BaseParser(ParserProtocol, abc.ABC):
         book-related folders or states.
         """
         pass
+
+    def _compile_ads_pattern(self) -> re.Pattern[str] | None:
+        """
+        Compile a regex pattern from the ADS list, or return None if no ADS.
+        """
+        if not self.ADS:
+            return None
+
+        return re.compile("|".join(map(re.escape, self.ADS)))
+
+    def _is_ad_line(self, line: str) -> bool:
+        """
+        Check if a line contains any ad text.
+
+        :param line: Single text line.
+        :return: True if line matches ad pattern, else False.
+        """
+        return bool(self._ad_pattern and self._ad_pattern.search(line))
+
+    def _filter_ads(self, lines: Iterable[str]) -> list[str]:
+        """
+        Filter out lines containing any ad text defined in ADS.
+
+        :param lines: Iterable of text lines (e.g. chapter content).
+        :return: List of lines with ads removed.
+        """
+        if not self._ad_pattern:
+            return list(lines)
+        return [line for line in lines if not self._ad_pattern.search(line)]
+
+    @classmethod
+    def _norm_space(cls, s: str, c: str = " ") -> str:
+        """
+        collapse any run of whitespace (incl. newlines, full-width spaces)
+
+        :param s: Input string to normalize.
+        :param c: Replacement character to use for collapsed whitespace.
+        """
+        return cls._SPACE_RE.sub(c, s).strip()
+
+    @staticmethod
+    def _first_str(xs: list[str], replaces: list[tuple[str, str]] | None = None) -> str:
+        replaces = replaces or []
+        value: str = xs[0].strip() if xs else ""
+        for replace in replaces:
+            old, new = replace
+            value = value.replace(old, new)
+        return value