novel-downloader 1.3.3__py3-none-any.whl → 1.4.1__py3-none-any.whl

novel_downloader/core/parsers/esjzone/main_parser.py

@@ -5,13 +5,13 @@ novel_downloader.core.parsers.esjzone.main_parser
 
 """
 
+import re
 from typing import Any
 
-from lxml import etree
-from lxml.etree import _Element
+from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 
 
 class EsjzoneParser(BaseParser):
@@ -40,7 +40,7 @@ class EsjzoneParser(BaseParser):
 
     def parse_book_info(
         self,
-        html_str: list[str],
+        html_list: list[str],
         **kwargs: Any,
     ) -> dict[str, Any]:
         """
@@ -49,12 +49,12 @@ class EsjzoneParser(BaseParser):
         注: 由于网站使用了多种不同的分卷格式, 已经尝试兼容常见情况,
         但仍可能存在未覆盖的 cases
 
-        :param html: Raw HTML of the book info page.
+        :param html_list: Raw HTML of the book info page.
         :return: Parsed metadata and chapter structure as a dictionary.
         """
-        if not html_str or self._is_forum_page(html_str):
+        if not html_list or self._is_forum_page(html_list):
             return {}
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_list[0])
         result: dict[str, Any] = {}
 
         result["book_name"] = self._get_text(tree, self._BOOK_NAME_XPATH)
@@ -75,8 +75,14 @@ class EsjzoneParser(BaseParser):
         volumes: list[dict[str, Any]] = []
         current_vol: dict[str, Any] = {}
 
+        def _is_garbage_title(name: str) -> bool:
+            stripped = name.strip()
+            return not stripped or bool(re.fullmatch(r"[\W_]+", stripped))
+
         def _start_volume(name: str) -> None:
             nonlocal current_vol
+            if _is_garbage_title(name):
+                return
             name = name.strip() or "未命名卷"
             if name == "未命名卷" and current_vol is not None:
                 return
@@ -94,7 +100,7 @@ class EsjzoneParser(BaseParser):
             tag = node.tag.lower()
 
             if tag == "details":
-                # ---- DETAILS‐based layout ----
+                # ---- DETAILS-based layout ----
                 summary = node.find("summary")
                 vol_name = summary.text if summary is not None else "未命名卷"
                 _start_volume(vol_name)
@@ -134,20 +140,20 @@ class EsjzoneParser(BaseParser):
 
     def parse_chapter(
         self,
-        html_str: list[str],
+        html_list: list[str],
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
         Parse a single chapter page and extract clean text or simplified HTML.
 
-        :param html: Raw HTML of the chapter page.
+        :param html_list: Raw HTML of the chapter page.
         :param chapter_id: Identifier of the chapter being parsed.
         :return: Cleaned chapter content as plain text or minimal HTML.
         """
-        if not html_str or self._is_forum_page(html_str):
+        if not html_list or self._is_forum_page(html_list):
             return None
-        tree = etree.HTML(html_str[0], parser=None)
+        tree = html.fromstring(html_list[0], parser=None)
 
         content_lines: list[str] = []
         content_nodes = tree.xpath(self._CHAPTER_CONTENT_NODES_XPATH)
@@ -198,7 +204,7 @@ class EsjzoneParser(BaseParser):
         if not html_str:
             return False
 
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_str[0])
         page_title = tree.xpath('string(//div[@class="page-title"]//h1)').strip()
         if page_title != "論壇":
             return False
@@ -208,7 +214,7 @@ class EsjzoneParser(BaseParser):
 
     @staticmethod
     def _get_text(
-        tree: _Element,
+        tree: html.HtmlElement,
         xpath: str,
         join: bool = False,
         clean_comma: bool = False,

novel_downloader/core/parsers/linovelib/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.linovelib
+---------------------------------------
+
+"""
+
+from .main_parser import LinovelibParser
+
+__all__ = ["LinovelibParser"]

novel_downloader/core/parsers/linovelib/main_parser.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.linovelib.main_parser
+---------------------------------------------------
+
+"""
+
+import json
+from itertools import islice
+from pathlib import PurePosixPath
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.models import ChapterDict
+from novel_downloader.utils.constants import LINOVELIB_FONT_MAP_PATH
+
+
+class LinovelibParser(BaseParser):
+    """ """
+
+    # Book info XPaths
+    _BOOK_NAME_XPATH = '//div[@class="book-info"]/h1[@class="book-name"]/text()'
+    _AUTHOR_XPATH = '//div[@class="au-name"]/a[1]/text()'
+    _COVER_URL_XPATH = '//div[contains(@class, "book-img")]//img/@src'
+    _UPDATE_TIME_XPATH = (
+        '//div[@class="nums"]/span[contains(text(), "最后更新")]/text()'  # noqa: E501
+    )
+    _SERIAL_STATUS_XPATH = '//div[@class="book-label"]/a[@class="state"]/text()'
+    _WORD_COUNT_XPATH = '//div[@class="nums"]/span[contains(text(), "字数")]/text()'
+    _SUMMARY_XPATH = '//div[contains(@class, "book-dec")]/p//text()'
+
+    _CHAPTERS_XPATH = '//div[@class="book-new-chapter"]/div[contains(@class, "tit")]/a'
+
+    # Chapter XPaths
+    _CHAPTER_TITLE_XPATH = "//div[@id='mlfy_main_text']/h1/text()"
+    _CHAPTER_CONTENT_NODES_XPATH = "//div[@id='TextContent']/*[self::p or self::img]"
+
+    _FONT_MAP: dict[str, str] = json.loads(
+        LINOVELIB_FONT_MAP_PATH.read_text(encoding="utf-8")
+    )  # 注意 json 前 3500 条的内容不必要不修改
+    _BLANK_SET: set[str] = set(islice(_FONT_MAP.values(), 3500))
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Parse a book info page and extract metadata and chapter structure.
+
+        :param html_list: Raw HTML of the book info page.
+        :return: Parsed metadata and chapter structure as a dictionary.
+        """
+        if not html_list:
+            return {}
+        info_tree = html.fromstring(html_list[0])
+        result: dict[str, Any] = {}
+
+        result["book_name"] = self._safe_xpath(info_tree, self._BOOK_NAME_XPATH)
+        result["author"] = self._safe_xpath(info_tree, self._AUTHOR_XPATH)
+        result["cover_url"] = self._safe_xpath(info_tree, self._COVER_URL_XPATH)
+        result["update_time"] = self._safe_xpath(
+            info_tree, self._UPDATE_TIME_XPATH, replace=("最后更新：", "")
+        )
+        result["serial_status"] = self._safe_xpath(info_tree, self._SERIAL_STATUS_XPATH)
+        result["word_count"] = self._safe_xpath(
+            info_tree, self._WORD_COUNT_XPATH, replace=("字数：", "")
+        )
+
+        result["summary"] = self._extract_intro(info_tree, self._SUMMARY_XPATH)
+
+        vol_pages = html_list[1:]
+        volumes: list[dict[str, Any]] = []
+        for vol_page in vol_pages:
+            vol_tree = html.fromstring(vol_page)
+            volume_cover = self._safe_xpath(vol_tree, self._COVER_URL_XPATH)
+            volume_name = self._safe_xpath(vol_tree, self._BOOK_NAME_XPATH)
+            update_time = self._safe_xpath(
+                vol_tree, self._UPDATE_TIME_XPATH, replace=("最后更新：", "")
+            )
+            word_count = self._safe_xpath(
+                vol_tree, self._WORD_COUNT_XPATH, replace=("字数：", "")
+            )
+            volume_intro = self._extract_intro(vol_tree, self._SUMMARY_XPATH)
+
+            chapters = []
+            chapter_elements = vol_tree.xpath(self._CHAPTERS_XPATH)
+            for a in chapter_elements:
+                title = a.text.strip()
+                url = a.attrib.get("href", "").strip()
+                chap_path = PurePosixPath(url.rstrip("/"))
+                chapters.append(
+                    {"title": title, "url": url, "chapterId": chap_path.stem}
+                )
+
+            volumes.append(
+                {
+                    "volume_name": volume_name,
+                    "volume_cover": volume_cover,
+                    "update_time": update_time,
+                    "word_count": word_count,
+                    "volume_intro": volume_intro,
+                    "chapters": chapters,
+                }
+            )
+        result["volumes"] = volumes
+
+        return result
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        """
+        Parse chapter pages and extract clean text or simplified HTML.
+
+        :param html_list: Raw HTML of the chapter page.
+        :param chapter_id: Identifier of the chapter being parsed.
+        :return: Cleaned chapter content as plain text or minimal HTML.
+        """
+        if not html_list:
+            return None
+        title_text: str = ""
+        contents: list[str] = []
+        for curr_html in html_list:
+            is_encrypted = self._is_encrypted(curr_html)
+            tree = html.fromstring(curr_html)
+
+            if not title_text:
+                titles = tree.xpath(self._CHAPTER_TITLE_XPATH)
+                if titles:
+                    title_text = titles[0].strip()
+
+            content_container = tree.xpath("//div[@id='TextContent']")
+            if not content_container:
+                continue
+            container = content_container[0]
+            nodes = container.xpath("./p | ./img")
+            all_p = container.xpath("./p")
+            total_p = len(all_p)
+            p_counter = 0
+
+            for node in nodes:
+                tag = node.tag.lower()
+                if tag == "p":
+                    raw_text = "".join(node.xpath(".//text()")).strip()
+                    if not raw_text:
+                        continue
+
+                    if is_encrypted and p_counter == total_p - 2:
+                        raw_text = self._apply_font_map(raw_text)
+
+                    contents.append(raw_text)
+                    p_counter += 1
+
+                elif tag == "img":
+                    src = node.get("data-src") or node.get("src", "")
+                    src = src.strip()
+                    if src:
+                        contents.append(f'<img src="{src}" />')
+        return {
+            "id": chapter_id,
+            "title": title_text,
+            "content": "\n\n".join(contents),
+            "extra": {"site": "linovelib"},
+        }
+
+    def _safe_xpath(
+        self,
+        tree: html.HtmlElement,
+        path: str,
+        replace: tuple[str, str] | None = None,
+    ) -> str:
+        result = tree.xpath(path)
+        if not result:
+            return ""
+        value: str = result[0].strip()
+        if replace:
+            old, new = replace
+            value = value.replace(old, new)
+        return value
+
+    @staticmethod
+    def _extract_intro(tree: html.HtmlElement, xpath: str) -> str:
+        paragraphs = tree.xpath(xpath.replace("//text()", ""))
+        lines = []
+        for p in paragraphs:
+            text_segments = p.xpath(".//text()")
+            cleaned = [seg.strip() for seg in text_segments if seg.strip()]
+            lines.append("\n".join(cleaned))
+        return "\n\n".join(lines)
+
+    @staticmethod
+    def _is_encrypted(html: str) -> bool:
+        """
+        Determine whether HTML content likely uses encrypted or obfuscated fonts.
+        """
+        return "CSSStyleSheet" in html
+
+    @classmethod
+    def _apply_font_map(cls, text: str) -> str:
+        """
+        Apply font mapping to the input text,
+        skipping characters in blank set.
+        """
+        return "".join(cls._FONT_MAP.get(c, c) for c in text if c not in cls._BLANK_SET)

novel_downloader/core/parsers/qianbi/main_parser.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.parsers.biquge.main_parser
+novel_downloader.core.parsers.qianbi.main_parser
 ------------------------------------------------
 
 """
@@ -8,10 +8,10 @@ novel_downloader.core.parsers.biquge.main_parser
 from datetime import datetime
 from typing import Any
 
-from lxml import etree
+from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 
 
 class QianbiParser(BaseParser):
@@ -19,20 +19,20 @@ class QianbiParser(BaseParser):
 
     def parse_book_info(
         self,
-        html_str: list[str],
+        html_list: list[str],
         **kwargs: Any,
     ) -> dict[str, Any]:
         """
         Parse a book info page and extract metadata and chapter structure.
 
-        :param html: Raw HTML of the book info page.
+        :param html_list: Raw HTML of the book info pages.
         :return: Parsed metadata and chapter structure as a dictionary.
         """
-        if len(html_str) < 2:
+        if len(html_list) < 2:
             return {}
 
-        info_tree = etree.HTML(html_str[0])
-        catalog_tree = etree.HTML(html_str[1])
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
         result: dict[str, Any] = {}
 
         title = info_tree.xpath('//h1[@class="page-title"]/text()')
@@ -56,9 +56,7 @@ class QianbiParser(BaseParser):
             '//div[@class="novel-info-item novel-info-content"]/span'
         )
         if summary_node and summary_node[0] is not None:
-            result["summary"] = etree.tostring(
-                summary_node[0], encoding="unicode", method="text"
-            ).strip()
+            result["summary"] = summary_node[0].text_content().strip()
         else:
             result["summary"] = ""
 
@@ -85,6 +83,8 @@ class QianbiParser(BaseParser):
                 if a_tag:
                     title = a_tag[0].xpath(".//span/text()")
                     href = a_tag[0].attrib.get("href", "")
+                    if href == "javascript:cid(0)":
+                        href = ""
                     chapter_id = (
                         href.split("/")[-1].replace(".html", "") if href else ""
                     )
@@ -105,20 +105,20 @@ class QianbiParser(BaseParser):
 
     def parse_chapter(
         self,
-        html_str: list[str],
+        html_list: list[str],
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
         Parse a single chapter page and extract clean text or simplified HTML.
 
-        :param html: Raw HTML of the chapter page.
+        :param html_list: Raw HTML of the chapter page.
         :param chapter_id: Identifier of the chapter being parsed.
         :return: Cleaned chapter content as plain text or minimal HTML.
         """
-        if not html_str:
+        if not html_list:
             return None
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_list[0])
 
         paras = tree.xpath('//div[@class="article-content"]/p/text()')
         content_text = "\n\n".join(p.strip() for p in paras if p.strip())
@@ -131,6 +131,11 @@ class QianbiParser(BaseParser):
         volume = tree.xpath('//h3[@class="text-muted"]/text()')
         volume_text = volume[0].strip() if volume else ""
 
+        next_href = tree.xpath('//div[@class="footer"]/a[@class="f-right"]/@href')
+        next_chapter_id = (
+            next_href[0].split("/")[-1].replace(".html", "") if next_href else ""
+        )
+
         return {
             "id": chapter_id,
             "title": title_text,
@@ -138,5 +143,6 @@ class QianbiParser(BaseParser):
             "extra": {
                 "site": "qianbi",
                 "volume": volume_text,
+                "next_chapter_id": next_chapter_id,
             },
         }

novel_downloader/core/parsers/qidian/__init__.py

@@ -3,17 +3,8 @@
 novel_downloader.core.parsers.qidian
 ------------------------------------
 
-This package provides parsing implementations for the Qidian platform.
-
-Modules:
-- browser: Contains `QidianBrowserParser` for browser-rendered page parsing.
-- session: Contains `QidianSessionParser` for session page parsing.
 """
 
-from .browser import QidianBrowserParser
-from .session import QidianSessionParser
+from .main_parser import QidianParser
 
-__all__ = [
-    "QidianBrowserParser",
-    "QidianSessionParser",
-]
+__all__ = ["QidianParser"]

novel_downloader/core/parsers/qidian/book_info_parser.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.qidian.book_info_parser
+-----------------------------------------------------
+
+This module provides parsing of Qidian book info pages.
+
+It extracts metadata such as title, author, cover URL, update
+time, status, word count, summary, and volume-chapter structure.
+"""
+
+import logging
+from typing import Any
+
+from lxml import html
+
+logger = logging.getLogger(__name__)
+
+_AUTHOR_XPATH = (
+    'string(//div[contains(@class, "book-info")]//a[contains(@class, "writer")])'
+)
+
+
+def _chapter_url_to_id(url: str) -> str:
+    return url.rstrip("/").split("/")[-1]
+
+
+def _get_volume_name(
+    vol_elem: html.HtmlElement,
+) -> str:
+    """
+    Extracts the volume title from a <div class="volume"> element using lxml.
+    Ignores <a> tags, and extracts text from other elements.
+    """
+    h3_candidates = vol_elem.xpath(".//h3")
+    if not h3_candidates:
+        return ""
+    texts = vol_elem.xpath(".//h3//text()[not(ancestor::a)]")
+    full_text = "".join(texts).strip()
+    return full_text.split(chr(183))[0].strip()
+
+
+def parse_book_info(html_str: str) -> dict[str, Any]:
+    """
+    Extract metadata: title, author, cover_url, update_time, status,
+    word_count, summary, and volumes with chapters.
+
+    :param html_str: Raw HTML of the book info page.
+    :return: A dict containing book metadata.
+    """
+    info: dict[str, Any] = {}
+    try:
+        doc = html.fromstring(html_str)
+
+        book_name = doc.xpath('string(//h1/em[@id="bookName"])').strip()
+        info["book_name"] = book_name
+
+        author = doc.xpath(_AUTHOR_XPATH).strip()
+        info["author"] = author
+
+        cover_url = doc.xpath('string(//div[@class="book-img"]//img/@src)').strip()
+        info["cover_url"] = cover_url
+
+        update_raw = (
+            doc.xpath('string(//span[contains(@class, "update-time")])')
+            .replace("更新时间", "")
+            .strip()
+        )
+        info["update_time"] = update_raw
+
+        status = doc.xpath('string(//p[@class="tag"]/span[@class="blue"][1])').strip()
+        info["serial_status"] = status
+
+        tags = doc.xpath('//p[@class="tag"]/a[@class="red"]/text()')
+        info["tags"] = [t.strip() for t in tags if t.strip()]
+
+        wc_number = doc.xpath("string(//p[em and cite][1]/em[1])").strip()
+        wc_unit = doc.xpath("string(//p[em and cite][1]/cite[1])").strip()
+        info["word_count"] = (
+            (wc_number + wc_unit) if wc_number and wc_unit else "Unknown"
+        )
+
+        summary = doc.xpath('string(//p[@class="intro"])').strip()
+        info["summary_brief"] = summary
+
+        intro_list = doc.xpath('//div[@class="book-intro"]/p')[0]
+        detail_intro = "\n".join(intro_list.itertext()).strip()
+        info["summary"] = detail_intro
+
+        volumes = []
+        for vol_div in doc.xpath('//div[@class="volume-wrap"]/div[@class="volume"]'):
+            volume_name = _get_volume_name(vol_div)
+            chapters = []
+            for li in vol_div.xpath(".//li"):
+                a = li.xpath(".//a")[0] if li.xpath(".//a") else None
+                if a is None or "href" not in a.attrib:
+                    continue
+                href = a.attrib["href"].strip()
+                title = "".join(a.itertext()).strip()
+                chapters.append(
+                    {
+                        "title": title,
+                        "url": href,
+                        "chapterId": _chapter_url_to_id(href),
+                    }
+                )
+            volumes.append({"volume_name": volume_name, "chapters": chapters})
+        info["volumes"] = volumes
+
+    except Exception as e:
+        logger.warning("[Parser] Error parsing book info: %s", e)
+
+    return info