novel-downloader 1.3.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/core/parsers/qidian/{browser/chapter_encrypted.py → chapter_encrypted.py}

@@ -1,16 +1,10 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.parsers.qidian.browser.chapter_encrypted
---------------------------------------------------------------
+novel_downloader.core.parsers.qidian.chapter_encrypted
+------------------------------------------------------
 
 Support for parsing encrypted chapters from Qidian using font OCR mapping,
 CSS rules, and custom rendering logic.
-
-Includes:
-- Font downloading and caching
-- Encrypted paragraph extraction
-- Custom CSS parsing and layout restoration
-- Font-based OCR decryption and mapping
 """
 
 from __future__ import annotations
@@ -21,27 +15,30 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 import tinycss2
-from bs4 import BeautifulSoup, Tag
+from lxml import html
 
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 from novel_downloader.utils.network import download_font_file
 from novel_downloader.utils.text_utils import apply_font_mapping
 
-from ..shared import (
+from .utils import (
     extract_chapter_info,
     find_ssr_page_context,
+    get_decryptor,
+    vip_status,
 )
 
 if TYPE_CHECKING:
-    from .main_parser import QidianBrowserParser
+    from .main_parser import QidianParser
 
 logger = logging.getLogger(__name__)
 IGNORED_CLASS_LISTS = {"title", "review"}
+NON_CONTENT_KEYWORDS = {"旧版", "反馈", "扫码"}
 
 
 def parse_encrypted_chapter(
-    parser: QidianBrowserParser,
-    soup: BeautifulSoup,
+    parser: QidianParser,
+    html_str: str,
     chapter_id: str,
 ) -> ChapterDict | None:
     """
@@ -52,9 +49,8 @@ def parse_encrypted_chapter(
     3. Decode and save randomFont bytes; download fixedFont via download_font().
     4. Extract paragraph structures and save debug JSON.
     5. Parse CSS rules and save debug JSON.
-    6. Determine paragraph name prefixes and ending number; save debug text.
-    7. Render encrypted paragraphs, then run OCR font-mapping.
-    8. Extracts paragraph texts and formats them.
+    6. Render encrypted paragraphs, then run OCR font-mapping.
+    7. Extracts paragraph texts and formats them.
 
     :param html_str: Raw HTML content of the chapter page.
     :return: Formatted chapter text or empty string if not parsable.
@@ -62,13 +58,14 @@ def parse_encrypted_chapter(
     try:
         if not (parser._decode_font and parser._font_ocr):
             return None
-        ssr_data = find_ssr_page_context(soup)
+        ssr_data = find_ssr_page_context(html_str)
         chapter_info = extract_chapter_info(ssr_data)
         if not chapter_info:
             logger.warning(
                 "[Parser] ssr_chapterInfo not found for chapter '%s'", chapter_id
             )
             return None
+
         debug_base_dir: Path | None = None
         if parser._font_debug_dir:
             debug_base_dir = parser._font_debug_dir / chapter_id
@@ -79,7 +76,9 @@ def parse_encrypted_chapter(
         fixedFontWoff2_url = chapter_info["fixedFontWoff2"]
 
         title = chapter_info.get("chapterName", "Untitled")
-        chapter_id = chapter_info.get("chapterId", "")
+        raw_html = chapter_info.get("content", "")
+        chapter_id = chapter_info.get("chapterId", chapter_id)
+        fkp = chapter_info.get("fkp", "")
         author_say = chapter_info.get("authorSay", "")
         update_time = chapter_info.get("updateTime", "")
         update_timestamp = chapter_info.get("updateTimestamp", 0)
@@ -101,7 +100,26 @@ def parse_encrypted_chapter(
             raise ValueError("fixed_path is None: failed to download font")
 
         # Extract and render paragraphs from HTML with CSS rules
-        main_paragraphs = extract_paragraphs_recursively(soup, chapter_id)
+        main_paragraphs = extract_paragraphs_recursively(html_str, chapter_id)
+        if not main_paragraphs or contains_keywords(
+            main_paragraphs, NON_CONTENT_KEYWORDS
+        ):
+            if vip_status(ssr_data):
+                try:
+                    decryptor = get_decryptor()
+                    raw_html = decryptor.decrypt(
+                        raw_html,
+                        chapter_id,
+                        fkp,
+                        parser._fuid,
+                    )
+                except Exception as e:
+                    logger.error(
+                        "[Parser] decryption failed for '%s': %s", chapter_id, e
+                    )
+                    return None
+            main_paragraphs = extract_paragraphs_recursively(raw_html, chapter_id)
+
         if debug_base_dir:
             main_paragraphs_path = debug_base_dir / "main_paragraphs_debug.json"
             main_paragraphs_path.write_text(
@@ -117,23 +135,11 @@ def parse_encrypted_chapter(
                 encoding="utf-8",
             )
 
-        paragraph_names = parse_paragraph_names(paragraphs_rules)
-        end_number = parse_end_number(main_paragraphs, paragraph_names)
-        if debug_base_dir:
-            paragraphs_rules_path = debug_base_dir / "paragraph_names_debug.txt"
-            temp = f"names:\n{paragraph_names}\n\nend_number: {end_number}"
-            paragraphs_rules_path.write_text(
-                temp,
-                encoding="utf-8",
-            )
-        if not end_number:
-            logger.warning(
-                f"[Parser] No end_number found after parsing chapter '{chapter_id}'"
-            )
-            return None
-
+        end_number = parse_end_number(main_paragraphs, paragraphs_rules)
         paragraphs_str, refl_list = render_paragraphs(
-            main_paragraphs, paragraphs_rules, end_number
+            main_paragraphs,
+            paragraphs_rules,
+            end_number,
         )
         if debug_base_dir:
             paragraphs_str_path = debug_base_dir / f"{chapter_id}_debug.txt"
@@ -173,7 +179,7 @@ def parse_encrypted_chapter(
         )
         return {
             "id": str(chapter_id),
-            "title": title,
+            "title": str(title),
             "content": final_paragraphs_str,
             "extra": {
                 "author_say": author_say.strip() if author_say else "",
@@ -183,6 +189,7 @@ def parse_encrypted_chapter(
                 "word_count": word_count,
                 "seq": seq,
                 "volume": volume,
+                "encrypted": True,
             },
         }
 
@@ -194,48 +201,46 @@ def parse_encrypted_chapter(
 
 
 def extract_paragraphs_recursively(
-    soup: BeautifulSoup, chapter_id: str = ""
+    html_str: str,
+    chapter_id: str,
 ) -> list[dict[str, Any]]:
-    """
-    Extracts paragraph elements under <main id="c-{chapter_id}"> from HTML
-    and converts them to a nested data structure for further processing.
+    def parse_element(elem: html.HtmlElement) -> dict[str, Any]:
+        class_attr = elem.attrib.get("class", "")
+        class_list = class_attr.split() if isinstance(class_attr, str) else class_attr
+        if "review" in class_list:
+            return {}
+
+        # Build attrs with class as list
+        attrs = {k: v.split() if k == "class" else v for k, v in elem.attrib.items()}
+
+        node: dict[str, Any] = {
+            "tag": elem.tag,
+            "attrs": attrs,
+            "data": [],
+        }
 
-    :param html_str: Full HTML content.
-    :param chapter_id: ID used to locate <main id="c-{chapter_id}">.
+        # Append entire elem.text if present (no splitting)
+        if elem.text:
+            node["data"].append(elem.text)
 
-    :return list: List of parsed <p> paragraph data.
-    """
+        # Recurse into children
+        for child in elem.iterchildren(tag=None):
+            child_dict = parse_element(child)
+            if child_dict:
+                node["data"].append(child_dict)
 
-    def parse_element(elem: Any) -> dict[str, Any] | None:
-        if not isinstance(elem, Tag):
-            return None
-        result = {"tag": elem.name, "attrs": dict(elem.attrs), "data": []}
-        for child in elem.contents:
-            if isinstance(child, Tag):
-                parsed = parse_element(child)
-                if parsed:
-                    result["data"].append(parsed)
-            else:
-                text = child
-                if text:
-                    result["data"].append(text)
-        return result
-
-    if chapter_id:
-        main_id = f"c-{chapter_id}"
-        main_tag = soup.find("main", id=main_id)
-        if not isinstance(main_tag, Tag):
-            return []
-    else:
-        main_tag = soup
-
-    result = []
-    for p in main_tag.find_all("p"):
-        parsed_p = parse_element(p)
-        if parsed_p:
-            result.append(parsed_p)
-
-    return result
+            # Append entire tail string (no split)
+            if child.tail:
+                node["data"].append(child.tail)
+
+        return node
+
+    tree = html.fromstring(html_str)
+
+    # Try to find <main id="c-{chapter_id}">
+    main_elem = tree.xpath(f'//main[@id="c-{chapter_id}"]')
+    search_root = main_elem[0] if main_elem else tree
+    return [parse_element(p) for p in search_root.findall(".//p")]
 
 
 def parse_rule(css_str: str) -> dict[str, Any]:
@@ -317,68 +322,10 @@ def parse_rule(css_str: str) -> dict[str, Any]:
     return {"rules": rules, "orders": orders}
 
 
-def parse_paragraph_names(rules: dict[str, Any]) -> set[str]:
-    """
-    Extract all paragraph selector names from parsed rules, excluding "sy".
-    """
-    paragraph_names = set()
-    for group, group_rules in rules.get("rules", {}).items():
-        if group == "sy":
-            continue
-        paragraph_names.update(group_rules.keys())
-    return paragraph_names
-
-
-def parse_end_number(
-    main_paragraphs: list[dict[str, Any]], paragraph_names: set[str]
-) -> int | None:
-    """
-    Find the most frequent numeric suffix from tag names
-    matched by given paragraph prefixes.
-    """
-    end_numbers: dict[int, int] = {}
-    sorted_names = sorted(paragraph_names, key=len, reverse=True)
-
-    def rec_parse(item: list[Any] | dict[str, Any]) -> None:
-        if isinstance(item, list):
-            for element in item:
-                rec_parse(element)
-        elif isinstance(item, dict):
-            tag = item.get("tag")
-            if isinstance(tag, str):
-                for prefix in sorted_names:
-                    if tag.startswith(prefix):
-                        remain = tag[len(prefix) :]
-                        if remain.isdigit():
-                            num = int(remain)
-                            end_numbers[num] = end_numbers.get(num, 0) + 1
-                        break
-            for val in item.values():
-                if isinstance(val, (list | dict)):
-                    rec_parse(val)
-
-    rec_parse(main_paragraphs)
-
-    if not end_numbers:
-        logger.warning("[Parser] No valid ending numbers found")
-        return None
-
-    sorted_numbers = sorted(
-        end_numbers.items(), key=lambda x: (x[1], x[0]), reverse=True
-    )
-
-    logger.debug(
-        "[Parser] Top 3 end numbers:\n%s",
-        "\n".join(f"{n}: {c}" for n, c in sorted_numbers[:3]),
-    )
-
-    return sorted_numbers[0][0]
-
-
 def render_paragraphs(
     main_paragraphs: list[dict[str, Any]],
     rules: dict[str, Any],
-    end_number: int,
+    end_number: str = "",
 ) -> tuple[str, list[str]]:
     """
     Applies the parsed CSS rules to the paragraph structure and
@@ -392,7 +339,6 @@ def render_paragraphs(
     :param rules: A dictionary with keys 'orders' and 'rules', parsed from CSS.
                   - rules['orders']: List of (selector, id) tuples.
                   - rules['rules']: Nested dict containing transformation rules.
-    :param end_number: HTML tag suffix (e.g. span123 -> 123).
 
     :return:
         - A reconstructed paragraph string with line breaks.
@@ -488,3 +434,84 @@ def render_paragraphs(
         paragraphs_str += "\n\n"
 
     return paragraphs_str, refl_list
+
+
+def parse_paragraph_names(rules: dict[str, Any]) -> set[str]:
+    """
+    Extract all paragraph selector names from parsed rules, excluding "sy".
+    """
+    paragraph_names = set()
+    for group, group_rules in rules.get("rules", {}).items():
+        if group == "sy":
+            continue
+        paragraph_names.update(group_rules.keys())
+    return paragraph_names
+
+
+def parse_end_number(
+    main_paragraphs: list[dict[str, Any]],
+    rules: dict[str, Any],
+) -> str:
+    """
+    Find the most frequent numeric suffix from tag names
+    matched by given paragraph prefixes.
+    """
+    paragraph_names = parse_paragraph_names(rules)
+    end_numbers: dict[int, int] = {}
+    prefix_hits = 0
+    sorted_names = sorted(paragraph_names, key=len, reverse=True)
+
+    def rec_parse(item: list[Any] | dict[str, Any]) -> None:
+        nonlocal prefix_hits
+        if isinstance(item, list):
+            for element in item:
+                rec_parse(element)
+        elif isinstance(item, dict):
+            tag = item.get("tag")
+            if isinstance(tag, str):
+                for prefix in sorted_names:
+                    if tag.startswith(prefix):
+                        prefix_hits += 1
+                        remain = tag[len(prefix) :]
+                        if remain.isdigit():
+                            num = int(remain)
+                            end_numbers[num] = end_numbers.get(num, 0) + 1
+                        break
+            for val in item.values():
+                if isinstance(val, (list | dict)):
+                    rec_parse(val)
+
+    rec_parse(main_paragraphs)
+
+    if not end_numbers:
+        logger.debug("[Parser] No valid ending numbers found")
+        return ""
+
+    sorted_numbers = sorted(
+        end_numbers.items(), key=lambda x: (x[1], x[0]), reverse=True
+    )
+
+    logger.debug(
+        "[Parser] Top 3 end numbers:\n%s",
+        "\n".join(f"{n}: {c}" for n, c in sorted_numbers[:3]),
+    )
+    most_common_number, most_common_count = sorted_numbers[0]
+    if most_common_count <= prefix_hits / 2:
+        logger.debug(
+            "[Parser] Top number (%s) does not exceed 50%% threshold: %d of %d",
+            most_common_number,
+            most_common_count,
+            prefix_hits,
+        )
+        return ""
+
+    return str(most_common_number)
+
+
+def contains_keywords(paragraphs: list[dict[str, Any]], keywords: set[str]) -> bool:
+    for para in paragraphs:
+        data = para.get("data", [])
+        for item in data:
+            if isinstance(item, str) and any(kw in item for kw in keywords):
+                return True
+    return False

novel_downloader/core/parsers/qidian/chapter_normal.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.qidian.chapter_normal
+---------------------------------------------------
+
+Parser logic for extracting readable text from Qidian chapters
+that use plain (non-encrypted) browser-rendered HTML.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from lxml import html
+
+from novel_downloader.models import ChapterDict
+
+from .utils import (
+    extract_chapter_info,
+    find_ssr_page_context,
+    get_decryptor,
+    vip_status,
+)
+
+if TYPE_CHECKING:
+    from .main_parser import QidianParser
+
+logger = logging.getLogger(__name__)
+
+
+def parse_normal_chapter(
+    parser: QidianParser,
+    html_str: str,
+    chapter_id: str,
+) -> ChapterDict | None:
+    """
+    Extract structured chapter info from a normal Qidian page.
+
+    :param html_str: Chapter HTML.
+    :param chapter_id: Chapter identifier (string).
+    :return: a dictionary with keys like 'id', 'title', 'content', etc.
+    """
+    try:
+        ssr_data = find_ssr_page_context(html_str)
+        chapter_info = extract_chapter_info(ssr_data)
+        if not chapter_info:
+            logger.warning(
+                "[Parser] ssr_chapterInfo not found for chapter '%s'", chapter_id
+            )
+            return None
+
+        title = chapter_info.get("chapterName", "Untitled")
+        raw_html = chapter_info.get("content", "")
+        chapter_id = chapter_info.get("chapterId", chapter_id)
+        fkp = chapter_info.get("fkp", "")
+        author_say = chapter_info.get("authorSay", "")
+        update_time = chapter_info.get("updateTime", "")
+        update_timestamp = chapter_info.get("updateTimestamp", 0)
+        modify_time = chapter_info.get("modifyTime", 0)
+        word_count = chapter_info.get("wordsCount", 0)
+        seq = chapter_info.get("seq", None)
+        volume = chapter_info.get("extra", {}).get("volumeName", "")
+
+        chapter_text = _parse_browser_paragraph(html_str)
+        if not chapter_text:
+            chapter_text = _parse_session_paragraph(
+                html_str=raw_html,
+                is_vip=vip_status(ssr_data),
+                chapter_id=chapter_id,
+                fkp=fkp,
+                fuid=parser._fuid,
+            )
+            if not chapter_text:
+                return None
+
+        return {
+            "id": str(chapter_id),
+            "title": title,
+            "content": chapter_text,
+            "extra": {
+                "author_say": author_say.strip() if author_say else "",
+                "updated_at": update_time,
+                "update_timestamp": update_timestamp,
+                "modify_time": modify_time,
+                "word_count": word_count,
+                "seq": seq,
+                "volume": volume,
+                "encrypted": False,
+            },
+        }
+    except Exception as e:
+        logger.warning(
+            "[Parser] parse error for normal chapter '%s': %s", chapter_id, e
+        )
+    return None
+
+
+def _parse_browser_paragraph(html_str: str) -> str:
+    try:
+        tree = html.fromstring(html_str)
+        main = tree.xpath('//div[@id="app"]//div[@id="reader-content"]//main')
+        if not main:
+            return ""
+        main = main[0]
+
+        content_spans = main.xpath('.//span[contains(@class, "content-text")]')
+
+        paragraph_texts = [
+            span.text_content().strip()
+            for span in content_spans
+            if span.text_content().strip()
+        ]
+
+        chapter_text = "\n\n".join(paragraph_texts)
+        return chapter_text
+
+    except Exception as e:
+        logger.error("[Parser] _parse_paragraph failed: %s", e)
+    return ""
+
+
+def _parse_session_paragraph(
+    html_str: str,
+    is_vip: bool,
+    chapter_id: str,
+    fkp: str,
+    fuid: str,
+) -> str:
+    try:
+        raw_html = html_str
+
+        if is_vip:
+            try:
+                decryptor = get_decryptor()
+                raw_html = decryptor.decrypt(raw_html, chapter_id, fkp, fuid)
+            except Exception as e:
+                logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
+                return ""
+
+        tree = html.fromstring(raw_html)
+        paras = tree.xpath(".//p")
+        paragraph_texts = [
+            p.text_content().strip() for p in paras if p.text_content().strip()
+        ]
+        return "\n\n".join(paragraph_texts)
+
+    except Exception as e:
+        logger.error("[Parser] _parse_paragraph failed: %s", e)
+    return ""

novel_downloader/core/parsers/qidian/{session/chapter_router.py → chapter_router.py}

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.parsers.qidian.session.chapter_router
------------------------------------------------------------
+novel_downloader.core.parsers.qidian.chapter_router
+---------------------------------------------------
 
-Routing logic for selecting the correct chapter parser for Qidian session pages.
+Routing logic for selecting the correct chapter parser for Qidian pages.
 """
 
 from __future__ import annotations
@@ -11,50 +11,50 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING
 
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 
-from ..shared import (
+from .chapter_normal import parse_normal_chapter
+from .utils import (
     can_view_chapter,
-    html_to_soup,
+    find_ssr_page_context,
     is_encrypted,
 )
-from .chapter_normal import parse_normal_chapter
 
 if TYPE_CHECKING:
-    from .main_parser import QidianSessionParser
+    from .main_parser import QidianParser
 
 logger = logging.getLogger(__name__)
 
 
 def parse_chapter(
-    parser: QidianSessionParser,
+    parser: QidianParser,
     html_str: str,
     chapter_id: str,
 ) -> ChapterDict | None:
     """
     Extract and return the formatted textual content of chapter.
 
-    :param parser: Instance of QidianSessionParser.
+    :param parser: Instance of QidianParser.
     :param html_str: Raw HTML content of the chapter page.
     :param chapter_id: Identifier of the chapter being parsed.
     :return: Formatted chapter text or empty string if not parsable.
     """
     try:
-        soup = html_to_soup(html_str)
+        ssr_data = find_ssr_page_context(html_str)
 
-        if not can_view_chapter(soup):
+        if not can_view_chapter(ssr_data):
             logger.warning(
                 "[Parser] Chapter '%s' is not purchased or inaccessible.", chapter_id
             )
             return None
 
-        if is_encrypted(soup):
+        if is_encrypted(ssr_data):
             if not parser._decode_font:
                 return None
             try:
                 from .chapter_encrypted import parse_encrypted_chapter
 
-                return parse_encrypted_chapter(parser, soup, chapter_id, parser._fuid)
+                return parse_encrypted_chapter(parser, html_str, chapter_id)
             except ImportError:
                 logger.warning(
                     "[Parser] Encrypted chapter '%s' requires extra dependencies.",
@@ -62,7 +62,7 @@ def parse_chapter(
                 )
                 return None
 
-        return parse_normal_chapter(soup, chapter_id, parser._fuid)
+        return parse_normal_chapter(parser, html_str, chapter_id)
     except Exception as e:
         logger.warning("[Parser] parse error for chapter '%s': %s", chapter_id, e)
     return None