novel-downloader 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/core/parsers/qidian/session/chapter_encrypted.py

@@ -1,443 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.qidian.session.chapter_encrypted
---------------------------------------------------------------
-
-Support for parsing encrypted chapters from Qidian using font OCR mapping,
-CSS rules, and custom rendering logic.
-
-Includes:
-- Font downloading and caching
-- Encrypted paragraph extraction
-- Custom CSS parsing and layout restoration
-- Font-based OCR decryption and mapping
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
-
-import tinycss2
-from bs4 import BeautifulSoup, Tag
-
-from novel_downloader.utils.chapter_storage import ChapterDict
-from novel_downloader.utils.network import download_font_file
-from novel_downloader.utils.text_utils import apply_font_mapping
-
-from ..shared import (
-    extract_chapter_info,
-    find_ssr_page_context,
-    html_to_soup,
-    vip_status,
-)
-from .node_decryptor import QidianNodeDecryptor
-
-if TYPE_CHECKING:
-    from .main_parser import QidianSessionParser
-
-logger = logging.getLogger(__name__)
-IGNORED_CLASS_LISTS = {"title", "review"}
-_decryptor: QidianNodeDecryptor | None = None
-
-
-def _get_decryptor() -> QidianNodeDecryptor:
-    """
-    Return the singleton QidianNodeDecryptor, initializing it on first use.
-    """
-    global _decryptor
-    if _decryptor is None:
-        _decryptor = QidianNodeDecryptor()
-    return _decryptor
-
-
-def parse_encrypted_chapter(
-    parser: QidianSessionParser,
-    soup: BeautifulSoup,
-    chapter_id: str,
-    fuid: str,
-) -> ChapterDict | None:
-    """
-    Extract and return the formatted textual content of an encrypted chapter.
-
-    Steps:
-    1. Load SSR JSON context for CSS, fonts, and metadata.
-    3. Decode and save randomFont bytes; download fixedFont via download_font().
-    4. Extract paragraph structures and save debug JSON.
-    5. Parse CSS rules and save debug JSON.
-    6. Render encrypted paragraphs, then run OCR font-mapping.
-    7. Extracts paragraph texts and formats them.
-
-    :param html_str: Raw HTML content of the chapter page.
-    :return: Formatted chapter text or empty string if not parsable.
-    """
-    try:
-        if not (parser._decode_font and parser._font_ocr):
-            return None
-        ssr_data = find_ssr_page_context(soup)
-        chapter_info = extract_chapter_info(ssr_data)
-        if not chapter_info:
-            logger.warning(
-                "[Parser] ssr_chapterInfo not found for chapter '%s'", chapter_id
-            )
-            return None
-        debug_base_dir: Path | None = None
-        if parser._font_debug_dir:
-            debug_base_dir = parser._font_debug_dir / chapter_id
-            debug_base_dir.mkdir(parents=True, exist_ok=True)
-
-        css_str = chapter_info["css"]
-        randomFont_str = chapter_info["randomFont"]
-        fixedFontWoff2_url = chapter_info["fixedFontWoff2"]
-
-        title = chapter_info.get("chapterName", "Untitled")
-        raw_html = chapter_info.get("content", "")
-        chapter_id = chapter_info.get("chapterId", "")
-        fkp = chapter_info.get("fkp", "")
-        author_say = chapter_info.get("authorSay", "")
-        update_time = chapter_info.get("updateTime", "")
-        update_timestamp = chapter_info.get("updateTimestamp", 0)
-        modify_time = chapter_info.get("modifyTime", 0)
-        word_count = chapter_info.get("wordsCount", 0)
-        seq = chapter_info.get("seq", None)
-        volume = chapter_info.get("extra", {}).get("volumeName", "")
-
-        if not raw_html:
-            logger.warning("[Parser] raw_html not found for chapter '%s'", chapter_id)
-            return None
-
-        # extract + save font
-        rf = json.loads(randomFont_str)
-        rand_path = parser._base_cache_dir / "randomFont.ttf"
-        rand_path.parent.mkdir(parents=True, exist_ok=True)
-        rand_path.write_bytes(bytes(rf["data"]))
-
-        fixed_path = download_font_file(
-            url=fixedFontWoff2_url, target_folder=parser._fixed_font_dir
-        )
-        if fixed_path is None:
-            raise ValueError("fixed_path is None: failed to download font")
-
-        # Extract and render paragraphs from HTML with CSS rules
-
-        if vip_status(soup):
-            try:
-                decryptor = _get_decryptor()
-                raw_html = decryptor.decrypt(
-                    raw_html,
-                    chapter_id,
-                    fkp,
-                    fuid,
-                )
-            except Exception as e:
-                logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
-                return None
-        main_paragraphs = extract_paragraphs_recursively(html_to_soup(raw_html))
-        if debug_base_dir:
-            main_paragraphs_path = debug_base_dir / "main_paragraphs_debug.json"
-            main_paragraphs_path.write_text(
-                json.dumps(main_paragraphs, ensure_ascii=False, indent=2),
-                encoding="utf-8",
-            )
-
-        paragraphs_rules = parse_rule(css_str)
-        if debug_base_dir:
-            paragraphs_rules_path = debug_base_dir / "paragraphs_rules_debug.json"
-            paragraphs_rules_path.write_text(
-                json.dumps(paragraphs_rules, ensure_ascii=False, indent=2),
-                encoding="utf-8",
-            )
-
-        paragraphs_str, refl_list = render_paragraphs(main_paragraphs, paragraphs_rules)
-        if debug_base_dir:
-            paragraphs_str_path = debug_base_dir / f"{chapter_id}_debug.txt"
-            paragraphs_str_path.write_text(paragraphs_str, encoding="utf-8")
-
-        # Run OCR + fallback mapping
-        char_set = {c for c in paragraphs_str if c not in {" ", "\n", "\u3000"}}
-        refl_set = set(refl_list)
-        char_set = char_set - refl_set
-        if debug_base_dir:
-            char_sets_path = debug_base_dir / "char_set_debug.txt"
-            temp = f"char_set:\n{char_set}\n\nrefl_set:\n{refl_set}"
-            char_sets_path.write_text(
-                temp,
-                encoding="utf-8",
-            )
-
-        mapping_result = parser._font_ocr.generate_font_map(
-            fixed_font_path=fixed_path,
-            random_font_path=rand_path,
-            char_set=char_set,
-            refl_set=refl_set,
-            chapter_id=chapter_id,
-        )
-        if debug_base_dir:
-            mapping_json_path = debug_base_dir / "font_mapping.json"
-            mapping_json_path.write_text(
-                json.dumps(mapping_result, ensure_ascii=False, indent=2),
-                encoding="utf-8",
-            )
-
-        # Reconstruct final readable text
-        original_text = apply_font_mapping(paragraphs_str, mapping_result)
-
-        final_paragraphs_str = "\n\n".join(
-            line.strip() for line in original_text.splitlines() if line.strip()
-        )
-        return {
-            "id": str(chapter_id),
-            "title": str(title),
-            "content": final_paragraphs_str,
-            "extra": {
-                "author_say": author_say.strip() if author_say else "",
-                "updated_at": update_time,
-                "update_timestamp": update_timestamp,
-                "modify_time": modify_time,
-                "word_count": word_count,
-                "seq": seq,
-                "volume": volume,
-            },
-        }
-
-    except Exception as e:
-        logger.warning(
-            "[Parser] parse error for encrypted chapter '%s': %s", chapter_id, e
-        )
-    return None
-
-
-def extract_paragraphs_recursively(
-    soup: BeautifulSoup, chapter_id: int = -1
-) -> list[dict[str, Any]]:
-    """
-    Extracts paragraph elements under <main id="c-{chapter_id}"> from HTML
-    and converts them to a nested data structure for further processing.
-
-    :param html_str: Full HTML content.
-    :param chapter_id: ID used to locate <main id="c-{chapter_id}">.
-
-    :return list: List of parsed <p> paragraph data.
-    """
-
-    def parse_element(elem: Any) -> dict[str, Any] | None:
-        if not isinstance(elem, Tag):
-            return None
-        result = {"tag": elem.name, "attrs": dict(elem.attrs), "data": []}
-        for child in elem.contents:
-            if isinstance(child, Tag):
-                parsed = parse_element(child)
-                if parsed:
-                    result["data"].append(parsed)
-            else:
-                text = child
-                if text:
-                    result["data"].append(text)
-        return result
-
-    if chapter_id > 0:
-        main_id = f"c-{chapter_id}"
-        main_tag = soup.find("main", id=main_id)
-        if not isinstance(main_tag, Tag):
-            return []
-    else:
-        main_tag = soup
-
-    result = []
-    for p in main_tag.find_all("p"):
-        parsed_p = parse_element(p)
-        if parsed_p:
-            result.append(parsed_p)
-
-    return result
-
-
-def parse_rule(css_str: str) -> dict[str, Any]:
-    """
-    Parse a CSS string and extract style rules for rendering.
-
-    Handles:
-    - font-size:0 (mark for deletion)
-    - scaleX(-1) (mark as mirrored)
-    - ::before / ::after with content or attr()
-    - class + tag selector mapping
-    - custom rendering order via 'order'
-
-    :param css_str: Raw CSS stylesheet string.
-    :return: Dict with "rules" and "orders" for rendering.
-    """
-
-    rules: dict[str, Any] = {}
-    orders = []
-
-    stylesheet = tinycss2.parse_stylesheet(
-        css_str, skip_comments=True, skip_whitespace=True
-    )
-
-    for rule in stylesheet:
-        if rule.type != "qualified-rule":
-            continue
-
-        selector = tinycss2.serialize(rule.prelude).strip()
-        declarations = tinycss2.parse_declaration_list(rule.content)
-
-        parsed = {}
-        order_val = None
-
-        for decl in declarations:
-            if decl.type != "declaration":
-                continue
-            name = decl.lower_name
-            value = tinycss2.serialize(decl.value).strip()
-
-            if name == "font-size" and value == "0":
-                if "::first-letter" in selector:
-                    parsed["delete-first"] = True
-                else:
-                    parsed["delete-all"] = True
-            elif name == "transform" and value.lower() == "scalex(-1)":
-                parsed["transform-x_-1"] = True
-            elif name == "order":
-                order_val = value
-            elif name == "content":
-                if "::after" in selector:
-                    if "attr(" in value:
-                        parsed["append-end-attr"] = value.split("attr(")[1].split(")")[
-                            0
-                        ]
-                    else:
-                        parsed["append-end-char"] = value.strip("\"'")
-                elif "::before" in selector:
-                    if "attr(" in value:
-                        parsed["append-start-attr"] = value.split("attr(")[1].split(
-                            ")"
-                        )[0]
-                    else:
-                        parsed["append-start-char"] = value.strip("\"'")
-
-        # Store in structure
-        if selector.startswith(".sy-"):
-            rules.setdefault("sy", {})[selector[1:]] = parsed
-        elif selector.startswith(".p") and " " in selector:
-            class_str, tag_part = selector.split(" ", 1)
-            class_str = class_str.lstrip(".")
-            tag_part = tag_part.split("::")[0]
-            rules.setdefault(class_str, {}).setdefault(tag_part, {}).update(parsed)
-
-        if order_val:
-            orders.append((selector, order_val))
-
-    orders.sort(key=lambda x: int(x[1]))
-    return {"rules": rules, "orders": orders}
-
-
-def render_paragraphs(
-    main_paragraphs: list[dict[str, Any]], rules: dict[str, Any]
-) -> tuple[str, list[str]]:
-    """
-    Applies the parsed CSS rules to the paragraph structure and
-    reconstructs the visible text.
-
-    Handles special class styles like .sy-*, text order control,
-    mirrored characters, etc.
-
-    :param main_paragraphs: A list of paragraph dictionaries, each with 'attrs'
-                            and 'data' fields representing structured content.
-    :param rules: A dictionary with keys 'orders' and 'rules', parsed from CSS.
-                  - rules['orders']: List of (selector, id) tuples.
-                  - rules['rules']: Nested dict containing transformation rules.
-
-    :return:
-        - A reconstructed paragraph string with line breaks.
-        - A list of mirrored (reflected) characters for later OCR processing.
-    """
-    orders: list[tuple[str, str]] = rules.get("orders", [])
-    rules = rules.get("rules", {})
-    refl_list: list[str] = []
-
-    def apply_rule(data: dict[str, Any], rule: dict[str, Any]) -> str:
-        if rule.get("delete-all", False):
-            return ""
-
-        curr_str = ""
-        if isinstance(data.get("data"), list) and data["data"]:
-            first_data = data["data"][0]
-            if isinstance(first_data, str):
-                curr_str += first_data
-
-        if rule.get("delete-first", False):
-            curr_str = "" if len(curr_str) <= 1 else curr_str[1:]
-
-        curr_str += rule.get("append-end-char", "")
-
-        attr_name = rule.get("append-end-attr", "")
-        if attr_name:
-            curr_str += data.get("attrs", {}).get(attr_name, "")
-
-        curr_str = rule.get("append-start-char", "") + curr_str
-
-        attr_name = rule.get("append-start-attr", "")
-        if attr_name:
-            curr_str = data.get("attrs", {}).get(attr_name, "") + curr_str
-
-        if rule.get("transform-x_-1", False):
-            refl_list.append(curr_str)
-        return curr_str
-
-    paragraphs_str = ""
-    for paragraph in main_paragraphs:
-        class_list = paragraph.get("attrs", {}).get("class", [])
-        p_class_str = next((c for c in class_list if c.startswith("p")), None)
-        curr_datas = paragraph.get("data", [])
-
-        ordered_cache = {}
-        for data in curr_datas:
-            # 文本节点直接加
-            if isinstance(data, str):
-                paragraphs_str += data
-                continue
-
-            if isinstance(data, dict):
-                tag = data.get("tag", "")
-                attrs = data.get("attrs", {})
-
-                # 跳过 span.review
-                if tag == "span" and "class" in attrs and "review" in attrs["class"]:
-                    continue
-
-                # sy 类型标签处理
-                if tag == "y":
-                    tag_class_list = attrs.get("class", [])
-                    tag_class = next(
-                        (c for c in tag_class_list if c.startswith("sy-")), None
-                    )
-
-                    if tag_class in rules.get("sy", {}):
-                        curr_rule = rules["sy"][tag_class]
-                        paragraphs_str += apply_rule(data, curr_rule)
-                    continue
-
-                if not p_class_str:
-                    if any(cls in IGNORED_CLASS_LISTS for cls in class_list):
-                        continue
-                    logger.debug(f"[parser] not find p_class_str: {class_list}")
-                    continue
-                # 普通标签处理，根据 orders 顺序匹配
-                for ord_selector, _ in orders:
-                    tag_name = f"{ord_selector}"
-                    if data.get("tag") != tag_name:
-                        continue
-                    curr_rule = rules.get(p_class_str, {}).get(ord_selector)
-                    curr_rule = curr_rule if curr_rule else {}
-                    ordered_cache[ord_selector] = apply_rule(data, curr_rule)
-                    break
-        # 最后按 orders 顺序拼接
-        for ord_selector, _ in orders:
-            if ord_selector in ordered_cache:
-                paragraphs_str += ordered_cache[ord_selector]
-
-        paragraphs_str += "\n\n"
-
-    return paragraphs_str, refl_list

novel_downloader/core/parsers/qidian/session/chapter_normal.py

@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.qidian.session.chapter_normal
------------------------------------------------------------
-
-Provides `parse_normal_chapter`, which will:
-
-  1. Extract SSR context from a “normal” (non-VIP) chapter page and format it.
-  2. Detect VIP/encrypted chapters and fall back to Node-based decryption
-     via `QidianNodeDecryptor`.
-"""
-
-import logging
-
-from bs4 import BeautifulSoup
-
-from novel_downloader.utils.chapter_storage import ChapterDict
-
-from ..shared import (
-    extract_chapter_info,
-    find_ssr_page_context,
-    html_to_soup,
-    vip_status,
-)
-from .node_decryptor import QidianNodeDecryptor
-
-logger = logging.getLogger(__name__)
-_decryptor: QidianNodeDecryptor | None = None
-
-
-def _get_decryptor() -> QidianNodeDecryptor:
-    """
-    Return the singleton QidianNodeDecryptor, initializing it on first use.
-    """
-    global _decryptor
-    if _decryptor is None:
-        _decryptor = QidianNodeDecryptor()
-    return _decryptor
-
-
-def parse_normal_chapter(
-    soup: BeautifulSoup,
-    chapter_id: str,
-    fuid: str,
-) -> ChapterDict | None:
-    """
-    Extract structured chapter info from a normal Qidian page.
-
-    :param soup:      A BeautifulSoup of the chapter HTML.
-    :param chapter_id: Chapter identifier (string).
-    :param fuid:      Fock user ID parameter from the page.
-    :return: a dictionary with keys like 'id', 'title', 'content', etc.
-    """
-    try:
-        ssr_data = find_ssr_page_context(soup)
-        chapter_info = extract_chapter_info(ssr_data)
-        if not chapter_info:
-            logger.warning(
-                "[Parser] ssr_chapterInfo not found for chapter '%s'", chapter_id
-            )
-            return None
-
-        title = chapter_info.get("chapterName", "Untitled")
-        raw_html = chapter_info.get("content", "")
-        chapter_id = chapter_info.get("chapterId", "")
-        fkp = chapter_info.get("fkp", "")
-        author_say = chapter_info.get("authorSay", "")
-        update_time = chapter_info.get("updateTime", "")
-        update_timestamp = chapter_info.get("updateTimestamp", 0)
-        modify_time = chapter_info.get("modifyTime", 0)
-        word_count = chapter_info.get("wordsCount", 0)
-        seq = chapter_info.get("seq", None)
-        volume = chapter_info.get("extra", {}).get("volumeName", "")
-
-        if not raw_html:
-            logger.warning("[Parser] raw_html not found for chapter '%s'", chapter_id)
-            return None
-
-        if vip_status(soup):
-            try:
-                decryptor = _get_decryptor()
-                raw_html = decryptor.decrypt(
-                    raw_html,
-                    chapter_id,
-                    fkp,
-                    fuid,
-                )
-            except Exception as e:
-                logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
-                return None
-
-        paras_soup = html_to_soup(raw_html)
-        paras = [p.get_text(strip=True) for p in paras_soup.find_all("p")]
-        chapter_text = "\n\n".join(paras)
-
-        return {
-            "id": str(chapter_id),
-            "title": title,
-            "content": chapter_text,
-            "extra": {
-                "author_say": author_say.strip() if author_say else "",
-                "updated_at": update_time,
-                "update_timestamp": update_timestamp,
-                "modify_time": modify_time,
-                "word_count": word_count,
-                "seq": seq,
-                "volume": volume,
-            },
-        }
-
-    except Exception as e:
-        logger.warning(
-            "[Parser] parse error for normal chapter '%s': %s", chapter_id, e
-        )
-    return None

novel_downloader/core/parsers/qidian/session/main_parser.py

@@ -1,128 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.parsers.qidian.session.main_parser
---------------------------------------------------------
-
-Main parser class for handling Qidian chapters rendered via a session.
-
-This module defines `QidianSessionParser`, a parser implementation that supports
-content extracted from dynamically rendered Qidian HTML pages.
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
-
-from novel_downloader.config.models import ParserConfig
-from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.utils.chapter_storage import ChapterDict
-from novel_downloader.utils.state import state_mgr
-
-from ..shared import (
-    is_encrypted,
-    parse_book_info,
-)
-from .chapter_router import parse_chapter
-
-if TYPE_CHECKING:
-    from novel_downloader.utils.fontocr import FontOCR
-
-
-class QidianSessionParser(BaseParser):
-    """
-    Parser for Qidian site using a session HTML workflow.
-    """
-
-    def __init__(self, config: ParserConfig):
-        """
-        Initialize the QidianBrowserParser with the given configuration.
-
-        :param config: ParserConfig object controlling:
-        """
-        super().__init__(config)
-
-        # Extract and store parser flags from config
-        self._decode_font: bool = config.decode_font
-        self._save_font_debug: bool = config.save_font_debug
-
-        self._fixed_font_dir: Path = self._base_cache_dir / "fixed_fonts"
-        self._fixed_font_dir.mkdir(parents=True, exist_ok=True)
-        self._font_debug_dir: Path | None = None
-
-        qd_cookies = state_mgr.get_cookies("qidian")
-        self._fuid: str = qd_cookies.get("ywguid", "")
-
-        self._font_ocr: FontOCR | None = None
-        if self._decode_font:
-            from novel_downloader.utils.fontocr import FontOCR
-
-            self._font_ocr = FontOCR(
-                cache_dir=self._base_cache_dir,
-                use_freq=config.use_freq,
-                use_ocr=config.use_ocr,
-                use_vec=config.use_vec,
-                batch_size=config.batch_size,
-                gpu_mem=config.gpu_mem,
-                gpu_id=config.gpu_id,
-                ocr_weight=config.ocr_weight,
-                vec_weight=config.vec_weight,
-                font_debug=config.save_font_debug,
-            )
-            self._font_debug_dir = self._base_cache_dir / "qidian" / "font_debug"
-            self._font_debug_dir.mkdir(parents=True, exist_ok=True)
-
-    def parse_book_info(
-        self,
-        html_str: list[str],
-        **kwargs: Any,
-    ) -> dict[str, Any]:
-        """
-        Parse a book info page and extract metadata and chapter structure.
-
-        :param html_str: Raw HTML of the book info page.
-        :return: Parsed metadata and chapter structure as a dictionary.
-        """
-        if not html_str:
-            return {}
-        return parse_book_info(html_str[0])
-
-    def parse_chapter(
-        self,
-        html_str: list[str],
-        chapter_id: str,
-        **kwargs: Any,
-    ) -> ChapterDict | None:
-        """
-        :param html: Raw HTML of the chapter page.
-        :param chapter_id: Identifier of the chapter being parsed.
-        :return: Cleaned chapter content as plain text.
-        """
-        if not html_str:
-            return None
-        return parse_chapter(self, html_str[0], chapter_id)
-
-    def is_encrypted(self, html_str: str) -> bool:
-        """
-        Return True if content is encrypted.
-
-        :param html: Raw HTML of the chapter page.
-        """
-        return is_encrypted(html_str)
-
-    def _init_cache_folders(self) -> None:
-        """
-        Prepare cache folders for plain/encrypted HTML and font debug data.
-        Folders are only created if corresponding debug/save flags are enabled.
-        """
-        base = self._base_cache_dir
-
-        # Font debug folder
-        if self._save_font_debug and self.book_id:
-            self._font_debug_dir = base / self.book_id / "font_debug"
-            self._font_debug_dir.mkdir(parents=True, exist_ok=True)
-        else:
-            self._font_debug_dir = None
-
-    def _on_book_id_set(self) -> None:
-        self._init_cache_folders()