novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/core/parsers/qidian_parser/session/chapter_router.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.session.chapter_router
+------------------------------------------------------------------
+
+Routing logic for selecting the correct chapter parser for Qidian session pages.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, Dict
+
+from ..shared import (
+    can_view_chapter,
+    html_to_soup,
+    is_encrypted,
+)
+from .chapter_normal import parse_normal_chapter
+
+if TYPE_CHECKING:
+    from .main_parser import QidianSessionParser
+
+logger = logging.getLogger(__name__)
+
+
+def parse_chapter(
+    parser: QidianSessionParser,
+    html_str: str,
+    chapter_id: str,
+) -> Dict[str, Any]:
+    """
+    Extract and return the formatted textual content of chapter.
+
+    :param parser: Instance of QidianSessionParser.
+    :param html_str: Raw HTML content of the chapter page.
+    :param chapter_id: Identifier of the chapter being parsed.
+    :return: Formatted chapter text or empty string if not parsable.
+    """
+    try:
+        soup = html_to_soup(html_str)
+
+        if not can_view_chapter(soup):
+            logger.warning(
+                "[Parser] Chapter '%s' is not purchased or inaccessible.", chapter_id
+            )
+            return {}
+
+        if is_encrypted(soup):
+            if not parser._decode_font:
+                return {}
+            try:
+                from .chapter_encrypted import parse_encrypted_chapter
+
+                return parse_encrypted_chapter(parser, soup, chapter_id, parser._fuid)
+            except ImportError:
+                logger.warning(
+                    "[Parser] Encrypted chapter '%s' requires extra dependencies.",
+                    chapter_id,
+                )
+                return {}
+
+        return parse_normal_chapter(soup, chapter_id, parser._fuid)
+    except Exception as e:
+        logger.warning("[Parser] parse error for chapter '%s': %s", chapter_id, e)
+        return {}

novel_downloader/core/parsers/qidian_parser/session/main_parser.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.session.main_parser
+---------------------------------------------------------------
+
+Main parser class for handling Qidian chapters rendered via a session.
+
+This module defines `QidianSessionParser`, a parser implementation that supports
+content extracted from dynamically rendered Qidian HTML pages.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from novel_downloader.config.models import ParserConfig
+from novel_downloader.core.parsers.base_parser import BaseParser
+from novel_downloader.utils.state import state_mgr
+
+from ..shared import (
+    is_encrypted,
+    parse_book_info,
+)
+from .chapter_router import parse_chapter
+
+if TYPE_CHECKING:
+    from novel_downloader.utils.fontocr import FontOCR
+
+
+class QidianSessionParser(BaseParser):
+    """
+    Parser for Qidian site using a session HTML workflow.
+    """
+
+    def __init__(self, config: ParserConfig):
+        """
+        Initialize the QidianBrowserParser with the given configuration.
+
+        :param config: ParserConfig object controlling:
+        """
+        super().__init__(config)
+
+        # Extract and store parser flags from config
+        self._decode_font: bool = config.decode_font
+        self._save_font_debug: bool = config.save_font_debug
+
+        self._fixed_font_dir: Path = self._base_cache_dir / "fixed_fonts"
+        self._fixed_font_dir.mkdir(parents=True, exist_ok=True)
+        self._font_debug_dir: Optional[Path] = None
+
+        qd_cookies = state_mgr.get_cookies("qidian")
+        self._fuid: str = qd_cookies.get("ywguid", "")
+
+        self._font_ocr: Optional[FontOCR] = None
+        if self._decode_font:
+            from novel_downloader.utils.fontocr import FontOCR
+
+            self._font_ocr = FontOCR(
+                cache_dir=self._base_cache_dir,
+                use_freq=config.use_freq,
+                use_ocr=config.use_ocr,
+                use_vec=config.use_vec,
+                batch_size=config.batch_size,
+                ocr_weight=config.ocr_weight,
+                vec_weight=config.vec_weight,
+                font_debug=config.save_font_debug,
+            )
+            self._font_debug_dir = self._base_cache_dir / "font_debug"
+            self._font_debug_dir.mkdir(parents=True, exist_ok=True)
+
+    def parse_book_info(self, html: str) -> Dict[str, Any]:
+        """
+        Parse a book info page and extract metadata and chapter structure.
+
+        :param html: Raw HTML of the book info page.
+        :return: Parsed metadata and chapter structure as a dictionary.
+        """
+        return parse_book_info(html)
+
+    def parse_chapter(self, html_str: str, chapter_id: str) -> Dict[str, Any]:
+        """
+        :param html: Raw HTML of the chapter page.
+        :param chapter_id: Identifier of the chapter being parsed.
+        :return: Cleaned chapter content as plain text.
+        """
+        return parse_chapter(self, html_str, chapter_id)
+
+    def is_encrypted(self, html_str: str) -> bool:
+        """
+        Return True if content is encrypted.
+
+        :param html: Raw HTML of the chapter page.
+        """
+        return is_encrypted(html_str)
+
+    def _init_cache_folders(self) -> None:
+        """
+        Prepare cache folders for plain/encrypted HTML and font debug data.
+        Folders are only created if corresponding debug/save flags are enabled.
+        """
+        base = self._base_cache_dir
+
+        # Font debug folder
+        if self._save_font_debug and self.book_id:
+            self._font_debug_dir = base / self.book_id / "font_debug"
+            self._font_debug_dir.mkdir(parents=True, exist_ok=True)
+        else:
+            self._font_debug_dir = None
+
+    def _on_book_id_set(self) -> None:
+        self._init_cache_folders()

novel_downloader/core/parsers/qidian_parser/session/node_decryptor.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.session.node_decryptor
+------------------------------------------------------------------
+
+Provides QidianNodeDecryptor, which ensures a Node.js environment,
+downloads or installs the required JS modules (Fock + decrypt script),
+and invokes a Node.js subprocess to decrypt Qidian chapter content.
+"""
+
+import json
+import logging
+import shutil
+import subprocess
+import uuid
+from pathlib import Path
+from typing import Union
+
+from novel_downloader.utils.constants import (
+    JS_SCRIPT_DIR,
+    QD_DECRYPT_SCRIPT_PATH,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class QidianNodeDecryptor:
+    """
+    A decryptor that uses Node.js plus Qidian's Fock JavaScript module
+    to decrypt encrypted chapter payloads.
+
+    On initialization, this class will:
+      1. Verify that `node` is on PATH.
+      2. Copy our bundled `qidian_decrypt_node.js` into `JS_SCRIPT_DIR`.
+      3. Download the remote Fock module JS if not already present.
+
+    Calling `decrypt()` will:
+      - Write a temp JSON input file with [ciphertext, chapter_id, fkp, fuid].
+      - Spawn `node qidian_decrypt_node.js <in> <out>`.
+      - Read and return the decrypted text.
+      - Clean up the temp files.
+    """
+
+    QIDIAN_FOCK_JS_URL: str = (
+        "https://cococdn.qidian.com/coco/s12062024/4819793b.qeooxh.js"
+    )
+    QIDIAN_FOCK_JS_PATH: Path = JS_SCRIPT_DIR / "4819793b.qeooxh.js"
+    QIDIAN_DECRYPT_SCRIPT_FILE: str = "qidian_decrypt_node.js"
+    QIDIAN_DECRYPT_SCRIPT_PATH: Path = JS_SCRIPT_DIR / QIDIAN_DECRYPT_SCRIPT_FILE
+
+    def __init__(self) -> None:
+        """
+        Prepare the script directory and verify that both Node.js
+        and the necessary JS files are available.
+        """
+        self.script_dir: Path = JS_SCRIPT_DIR
+        self.script_dir.mkdir(parents=True, exist_ok=True)
+        self.script_path: Path = self.QIDIAN_DECRYPT_SCRIPT_PATH
+        self._check_environment()
+
+    def _check_environment(self) -> None:
+        """
+        Ensure Node.js is installed, our decrypt script is copied from
+        package resources, and the Fock JS module is downloaded.
+
+        :raises EnvironmentError: if `node` is not on the system PATH.
+        """
+        # 1) Check Node.js
+        if not shutil.which("node"):
+            raise EnvironmentError("Node.js is not installed or not in PATH.")
+
+        # 2) Copy bundled decrypt script into place if missing
+        if not self.QIDIAN_DECRYPT_SCRIPT_PATH.exists():
+            try:
+                resource = QD_DECRYPT_SCRIPT_PATH
+                shutil.copyfile(str(resource), str(self.QIDIAN_DECRYPT_SCRIPT_PATH))
+                logger.info(
+                    "[decryptor] Copied decrypt script to %s",
+                    self.QIDIAN_DECRYPT_SCRIPT_PATH,
+                )
+            except Exception as e:
+                logger.error("[decryptor] Failed to copy decrypt script: %s", e)
+                raise
+
+        # 3) Download the Fock JS module from Qidian CDN if missing
+        if not self.QIDIAN_FOCK_JS_PATH.exists():
+            from novel_downloader.utils.network import download_js_file
+
+            try:
+                download_js_file(
+                    self.QIDIAN_FOCK_JS_URL,
+                    self.script_dir,
+                    on_exist="overwrite",
+                )
+                logger.info(
+                    "[decryptor] Downloaded Fock module to %s", self.QIDIAN_FOCK_JS_PATH
+                )
+            except Exception as e:
+                logger.error("[decryptor] Failed to download Fock JS module: %s", e)
+                raise
+
+    def decrypt(
+        self,
+        ciphertext: Union[str, bytes],
+        chapter_id: Union[str, int],
+        fkp: str,
+        fuid: str,
+    ) -> str:
+        """
+        Decrypt a chapter payload via our Node.js script.
+
+        :param ciphertext: Base64-encoded encrypted content (str or bytes).
+        :param chapter_id: The chapter's numeric ID.
+        :param fkp: Base64-encoded Fock key param from the page.
+        :param fuid: Fock user ID param from the page.
+        :return: The decrypted plain-text content.
+        :raises RuntimeError: if the Node.js subprocess exits with a non-zero code.
+        """
+        # Normalize inputs
+        cipher_str = (
+            ciphertext.decode("utf-8")
+            if isinstance(ciphertext, (bytes, bytearray))
+            else str(ciphertext)
+        )
+        chapter_str = str(chapter_id)
+
+        # Create unique temp file names
+        task_id = uuid.uuid4().hex
+        input_path = self.script_dir / f"input_{task_id}.json"
+        output_path = self.script_dir / f"output_{task_id}.txt"
+
+        try:
+            # Write arguments as JSON array
+            input_path.write_text(
+                json.dumps([cipher_str, chapter_str, fkp, fuid]),
+                encoding="utf-8",
+            )
+
+            logger.debug(
+                "[decryptor] Invoking Node.js: node %s %s %s",
+                self.script_path.name,
+                input_path.name,
+                output_path.name,
+            )
+
+            proc = subprocess.run(
+                ["node", self.script_path.name, input_path.name, output_path.name],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                cwd=str(self.script_dir),
+            )
+
+            if proc.returncode != 0:
+                raise RuntimeError(f"Node error: {proc.stderr.strip()}")
+
+            # Return decrypted content
+            return output_path.read_text(encoding="utf-8").strip()
+
+        finally:
+            # Clean up temp files
+            input_path.unlink(missing_ok=True)
+            output_path.unlink(missing_ok=True)

novel_downloader/core/parsers/qidian_parser/shared/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.shared
+--------------------------------------------------
+
+Shared parsing utilities for Qidian parser components.
+
+This subpackage provides common functions used across
+different Qidian parsing strategies. It encapsulates logic for:
+
+- Parsing the SSR-rendered page context and chapter metadata.
+- Determining access control and encryption status of chapters.
+- Basic HTML preprocessing and fallback parsing behavior.
+- Extracting structured book info from the main book page.
+"""
+
+from .book_info_parser import parse_book_info
+from .helpers import (
+    can_view_chapter,
+    extract_chapter_info,
+    find_ssr_page_context,
+    html_to_soup,
+    is_encrypted,
+    is_vip,
+    vip_status,
+)
+
+__all__ = [
+    "parse_book_info",
+    "html_to_soup",
+    "is_vip",
+    "can_view_chapter",
+    "is_encrypted",
+    "vip_status",
+    "find_ssr_page_context",
+    "extract_chapter_info",
+]

novel_downloader/core/parsers/qidian_parser/shared/book_info_parser.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.shared.book_info_parser
+-------------------------------------------------------------------
+
+This module provides parsing of Qidian book info pages.
+
+It extracts metadata such as title, author, cover URL, update
+time, status, word count, summary, and volume-chapter structure.
+"""
+
+import logging
+import re
+from typing import Any, Dict
+
+from bs4.element import Tag
+
+from .helpers import html_to_soup
+
+logger = logging.getLogger(__name__)
+
+
+def _chapter_url_to_id(url: str) -> str:
+    """
+    Extract chapterId as the last non-empty segment of the URL.
+    """
+    return url.rstrip("/").split("/")[-1]
+
+
+def _get_volume_name(vol_div: Tag) -> str:
+    """
+    Extracts the volume title from a <div class="volume"> element
+    """
+    h3 = vol_div.select_one("h3")
+    if not h3:
+        return ""
+    for a in h3.find_all("a"):
+        a.decompose()
+    text: str = h3.get_text(strip=True)
+    return text.split(chr(183))[0].strip()
+
+
+def parse_book_info(html_str: str) -> Dict[str, Any]:
+    """
+    Extract metadata: title, author, cover_url, update_time, status,
+    word_count, summary, and volumes with chapters.
+
+    :param html_str: Raw HTML of the book info page.
+    :return: A dict containing book metadata.
+    """
+    info: Dict[str, Any] = {}
+    try:
+        soup = html_to_soup(html_str)
+        info["book_name"] = soup.select_one("em#bookName").get_text(strip=True)
+        info["author"] = soup.select_one("a.writer").get_text(strip=True)
+        info["cover_url"] = soup.select_one("div.book-img img")["src"].strip()
+        info["update_time"] = (
+            soup.select_one("span.book-update-time")
+            .get_text(strip=True)
+            .replace("更新时间", "")
+            .strip()
+        )
+        info["serial_status"] = soup.select_one("span.blue").get_text(strip=True)
+        # word count via regex
+        match = re.search(
+            r"<em>([\d.]+)</em>\s*<cite>(.*?)字</cite>",
+            html_str,
+        )
+        if match:
+            info["word_count"] = match.group(1) + match.group(2) + "字"
+        else:
+            info["word_count"] = "Unknown"
+        info["summary"] = soup.select_one("div.book-intro p").get_text(
+            separator="\n", strip=True
+        )
+        # volumes
+        vols = []
+        for vol_div in soup.select("div.volume-wrap div.volume"):
+            name = _get_volume_name(vol_div)
+            chaps = []
+            for li in vol_div.select("li"):
+                a = li.select_one("a")
+                chaps.append(
+                    {
+                        "title": a.get_text(strip=True),
+                        "url": a["href"].strip(),
+                        "chapterId": _chapter_url_to_id(a["href"]),
+                    }
+                )
+            vols.append({"volume_name": name, "chapters": chaps})
+        info["volumes"] = vols
+    except Exception as e:
+        logger.warning("[Parser] Error parsing book info: %s", e)
+    return info

novel_downloader/core/parsers/qidian_parser/shared/helpers.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.parsers.qidian_parser.shared.helpers
+----------------------------------------------------------
+
+Shared utility functions for parsing Qidian browser-rendered pages.
+
+This module provides reusable helpers to:
+- Convert HTML into BeautifulSoup objects with fallback.
+- Extract SSR-rendered JSON page context and structured chapter metadata.
+- Identify VIP chapters, encrypted content, and viewability conditions.
+"""
+
+import json
+import logging
+from typing import Any, Dict, Union
+
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+
+def html_to_soup(html_str: str) -> BeautifulSoup:
+    """
+    Convert an HTML string to a BeautifulSoup object with fallback.
+
+    :param html_str: Raw HTML string.
+    :return: Parsed BeautifulSoup object.
+    """
+    try:
+        return BeautifulSoup(html_str, "lxml")
+    except Exception as e:
+        logger.warning("[Parser] lxml parse failed, falling back: %s", e)
+        return BeautifulSoup(html_str, "html.parser")
+
+
+def is_vip(html_str: str) -> bool:
+    """
+    Return True if page indicates VIP‐only content.
+
+    :param html_str: Raw HTML string.
+    """
+    markers = ["这是VIP章节", "需要订阅", "订阅后才能阅读"]
+    return any(m in html_str for m in markers)
+
+
+def vip_status(soup: BeautifulSoup) -> bool:
+    """
+    :param soup: Parsed BeautifulSoup object of the HTML page.
+    :return: True if VIP, False otherwise.
+    """
+    ssr_data = find_ssr_page_context(soup)
+    chapter_info = extract_chapter_info(ssr_data)
+    vip_flag = chapter_info.get("vipStatus", 0)
+    fens_flag = chapter_info.get("fEnS", 0)
+    return bool(vip_flag == 1 and fens_flag != 0)
+
+
+def can_view_chapter(soup: BeautifulSoup) -> bool:
+    """
+    Return True if the chapter is viewable by the current user.
+
+    A chapter is not viewable if it is marked as VIP
+    and has not been purchased.
+
+    :param soup: Parsed BeautifulSoup object of the HTML page.
+    :return: True if viewable, False otherwise.
+    """
+    ssr_data = find_ssr_page_context(soup)
+    chapter_info = extract_chapter_info(ssr_data)
+
+    is_buy = chapter_info.get("isBuy", 0)
+    vip_status = chapter_info.get("vipStatus", 0)
+
+    return not (vip_status == 1 and is_buy == 0)
+
+
+def is_encrypted(content: Union[str, BeautifulSoup]) -> bool:
+    """
+    Return True if content is encrypted.
+
+    Chapter Encryption Status (cES):
+    - 0: 内容是'明文'
+    - 2: 字体加密
+
+    :param content: HTML content, either as a raw string or a BeautifulSoup object.
+    :return: True if encrypted marker is found, else False.
+    """
+    # main = soup.select_one("div#app div#reader-content main")
+    # return bool(main and "r-font-encrypt" in main.get("class", []))
+    # Normalize to BeautifulSoup
+    soup = html_to_soup(content) if isinstance(content, str) else content
+
+    ssr_data = find_ssr_page_context(soup)
+    chapter_info = extract_chapter_info(ssr_data)
+    return int(chapter_info.get("cES", 0)) == 2
+
+
+def find_ssr_page_context(soup: BeautifulSoup) -> Dict[str, Any]:
+    """
+    Extract SSR JSON from <script id="vite-plugin-ssr_pageContext">.
+    """
+    try:
+        tag = soup.find("script", id="vite-plugin-ssr_pageContext")
+        if tag and tag.string:
+            data: Dict[str, Any] = json.loads(tag.string.strip())
+            return data
+    except Exception as e:
+        logger.warning("[Parser] SSR JSON parse error: %s", e)
+    return {}
+
+
+def extract_chapter_info(ssr_data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Extract the 'chapterInfo' dictionary from the SSR page context.
+
+    This handles nested key access and returns an empty dict if missing.
+
+    :param ssr_data: The full SSR data object from _find_ssr_page_context().
+    :return: A dict with chapter metadata such as chapterName, authorSay, etc.
+    """
+    try:
+        page_context = ssr_data.get("pageContext", {})
+        page_props = page_context.get("pageProps", {})
+        page_data = page_props.get("pageData", {})
+        chapter_info = page_data.get("chapterInfo", {})
+
+        assert isinstance(chapter_info, dict)
+        return chapter_info
+    except Exception as e:
+        logger.warning("[Parser] Failed to extract chapterInfo: %s", e)
+        return {}

novel_downloader/core/requesters/__init__.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters
+--------------------------------
+
+This package provides requester implementations for different novel platforms.
+Each submodule corresponds to a specific site and encapsulates the logic needed
+to perform network interactions, such as logging in, sending requests,
+or interacting with browser/session-based sources.
+
+Subpackages:
+- common_requester: Handles all common-site requesting logic.
+- qidian_requester: Handles all Qidian-related requesting logic.
+"""
+
+from .common_requester import (
+    CommonAsyncSession,
+    CommonSession,
+)
+from .qidian_requester import (
+    QidianBrowser,
+    QidianSession,
+)
+
+__all__ = [
+    "CommonAsyncSession",
+    "CommonSession",
+    "QidianBrowser",
+    "QidianSession",
+]