novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/qidian/chapter_normal.py

@@ -15,7 +15,7 @@ from typing import TYPE_CHECKING
 from lxml import html
 
 from novel_downloader.models import ChapterDict
-from novel_downloader.utils.text_utils import truncate_half_lines
+from novel_downloader.utils import truncate_half_lines
 
 from .utils import (
     extract_chapter_info,
@@ -65,17 +65,15 @@ def parse_normal_chapter(
         seq = chapter_info.get("seq", None)
         volume = chapter_info.get("extra", {}).get("volumeName", "")
 
-        chapter_text = _parse_browser_paragraph(html_str)
+        chapter_text = _parse_paragraph(
+            html_str=raw_html,
+            is_vip=vip_status(ssr_data),
+            chapter_id=chapter_id,
+            fkp=fkp,
+            fuid=parser._fuid,
+        )
         if not chapter_text:
-            chapter_text = _parse_session_paragraph(
-                html_str=raw_html,
-                is_vip=vip_status(ssr_data),
-                chapter_id=chapter_id,
-                fkp=fkp,
-                fuid=parser._fuid,
-            )
-            if not chapter_text:
-                return None
+            return None
 
         if parser._use_truncation and duplicated:
             chapter_text = truncate_half_lines(chapter_text)
@@ -103,55 +101,26 @@ def parse_normal_chapter(
     return None
 
 
-def _parse_browser_paragraph(html_str: str) -> str:
-    try:
-        tree = html.fromstring(html_str)
-        main = tree.xpath('//div[@id="app"]//div[@id="reader-content"]//main')
-        if not main:
-            return ""
-        main = main[0]
-
-        content_spans = main.xpath('.//span[contains(@class, "content-text")]')
-
-        paragraph_texts = [
-            span.text_content().strip()
-            for span in content_spans
-            if span.text_content().strip()
-        ]
-
-        chapter_text = "\n\n".join(paragraph_texts)
-        return chapter_text
-
-    except Exception as e:
-        logger.error("[Parser] _parse_paragraph failed: %s", e)
-    return ""
-
-
-def _parse_session_paragraph(
+def _parse_paragraph(
     html_str: str,
     is_vip: bool,
     chapter_id: str,
     fkp: str,
     fuid: str,
 ) -> str:
-    try:
-        raw_html = html_str
-
-        if is_vip:
-            try:
-                decryptor = get_decryptor()
-                raw_html = decryptor.decrypt(raw_html, chapter_id, fkp, fuid)
-            except Exception as e:
-                logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
-                return ""
-
-        tree = html.fromstring(raw_html)
-        paras = tree.xpath(".//p")
-        paragraph_texts = [
-            p.text_content().strip() for p in paras if p.text_content().strip()
-        ]
-        return "\n\n".join(paragraph_texts)
+    raw_html = html_str
+
+    if is_vip:
+        try:
+            decryptor = get_decryptor()
+            raw_html = decryptor.decrypt(raw_html, chapter_id, fkp, fuid)
+        except Exception as e:
+            logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
+            return ""
 
-    except Exception as e:
-        logger.error("[Parser] _parse_paragraph failed: %s", e)
-    return ""
+    tree = html.fromstring(raw_html)
+    paras = tree.xpath(".//p")
+    paragraph_texts = [
+        p.text_content().strip() for p in paras if p.text_content().strip()
+    ]
+    return "\n".join(paragraph_texts)

novel_downloader/core/parsers/qidian/main_parser.py

@@ -10,12 +10,17 @@ from __future__ import annotations
 
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.models import ChapterDict, ParserConfig
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ParserConfig,
+)
 from novel_downloader.utils.constants import DATA_DIR
-from novel_downloader.utils.cookies import find_cookie_value
+from novel_downloader.utils.cookies import get_cookie_value
 
 from .book_info_parser import parse_book_info
 from .chapter_router import parse_chapter
@@ -23,13 +28,13 @@ from .utils import is_encrypted
 
 logger = logging.getLogger(__name__)
 
-if TYPE_CHECKING:
-    from novel_downloader.utils.fontocr import FontOCR
-
 
+@register_parser(
+    site_keys=["qidian", "qd"],
+)
 class QidianParser(BaseParser):
     """
-    Parser for Qidian site.
+    Parser for 起点中文网 site.
     """
 
     def __init__(
@@ -44,50 +49,20 @@ class QidianParser(BaseParser):
         """
         super().__init__(config)
 
-        # Extract and store parser flags from config
-        self._use_truncation = config.use_truncation
-        self._decode_font: bool = config.decode_font
-        self._save_font_debug: bool = config.save_font_debug
-
         self._fixed_font_dir: Path = self._base_cache_dir / "fixed_fonts"
         self._fixed_font_dir.mkdir(parents=True, exist_ok=True)
-        self._font_debug_dir: Path | None = None
+        self._debug_dir: Path = Path.cwd() / "debug"
 
         state_files = [
-            DATA_DIR / "qidian" / "browser_state.cookies",
             DATA_DIR / "qidian" / "session_state.cookies",
         ]
-        self._fuid: str = fuid or find_cookie_value(state_files, "ywguid")
-
-        self._font_ocr: FontOCR | None = None
-        if self._decode_font:
-            try:
-                from novel_downloader.utils.fontocr import FontOCR
-            except ImportError:
-                logger.warning(
-                    "[QidianParser] FontOCR not available, font decoding will skip"
-                )
-            else:
-                self._font_ocr = FontOCR(
-                    cache_dir=self._base_cache_dir,
-                    use_freq=config.use_freq,
-                    use_ocr=config.use_ocr,
-                    use_vec=config.use_vec,
-                    batch_size=config.batch_size,
-                    gpu_mem=config.gpu_mem,
-                    gpu_id=config.gpu_id,
-                    ocr_weight=config.ocr_weight,
-                    vec_weight=config.vec_weight,
-                    font_debug=config.save_font_debug,
-                )
-                self._font_debug_dir = self._base_cache_dir / "qidian" / "font_debug"
-                self._font_debug_dir.mkdir(parents=True, exist_ok=True)
+        self._fuid: str = fuid or get_cookie_value(state_files, "ywguid")
 
     def parse_book_info(
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Parse a book info page and extract metadata and chapter structure.
 
@@ -95,7 +70,7 @@ class QidianParser(BaseParser):
         :return: Parsed metadata and chapter structure as a dictionary.
         """
         if not html_list:
-            return {}
+            return None
         return parse_book_info(html_list[0])
 
     def parse_chapter(
@@ -121,19 +96,6 @@ class QidianParser(BaseParser):
         """
         return is_encrypted(html_str)
 
-    def _init_cache_folders(self) -> None:
-        """
-        Prepare cache folders for plain/encrypted HTML and font debug data.
-        Folders are only created if corresponding debug/save flags are enabled.
-        """
-        base = self._base_cache_dir
-
-        # Font debug folder
-        if self._save_font_debug and self.book_id:
-            self._font_debug_dir = base / self.book_id / "font_debug"
-            self._font_debug_dir.mkdir(parents=True, exist_ok=True)
-        else:
-            self._font_debug_dir = None
-
-    def _on_book_id_set(self) -> None:
-        self._init_cache_folders()
+    @property
+    def save_font_debug(self) -> bool:
+        return self._config.save_font_debug

novel_downloader/core/parsers/qidian/utils/__init__.py

@@ -3,19 +3,9 @@
 novel_downloader.core.parsers.qidian.utils
 ------------------------------------------
 
+Utility functions and helpers for parsing and decrypting Qidian novel pages
 """
 
-from .helpers import (
-    can_view_chapter,
-    extract_chapter_info,
-    find_ssr_page_context,
-    is_duplicated,
-    is_encrypted,
-    is_restricted_page,
-    vip_status,
-)
-from .node_decryptor import QidianNodeDecryptor, get_decryptor
-
 __all__ = [
     "find_ssr_page_context",
     "extract_chapter_info",
@@ -27,3 +17,14 @@ __all__ = [
     "QidianNodeDecryptor",
     "get_decryptor",
 ]
+
+from .helpers import (
+    can_view_chapter,
+    extract_chapter_info,
+    find_ssr_page_context,
+    is_duplicated,
+    is_encrypted,
+    is_restricted_page,
+    vip_status,
+)
+from .node_decryptor import QidianNodeDecryptor, get_decryptor

novel_downloader/core/parsers/qidian/utils/decryptor_fetcher.py

@@ -9,6 +9,11 @@ GitHub releases.
 
 from __future__ import annotations
 
+__all__ = [
+    "ensure_decryptor",
+    "RELEASE_VERSION",
+]
+
 import hashlib
 import platform
 import stat
@@ -20,7 +25,7 @@ import requests
 from novel_downloader.utils.constants import JS_SCRIPT_DIR
 
 DEST_ROOT: Final[Path] = JS_SCRIPT_DIR
-GITHUB_OWNER: Final = "BowenZ217"
+GITHUB_OWNER: Final = "saudadez21"
 GITHUB_REPO: Final = "qidian-decryptor"
 RELEASE_VERSION: Final = "v1.0.1"
 BASE_URL: Final = f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}/releases/download/{RELEASE_VERSION}"
@@ -137,9 +142,3 @@ def _make_executable(p: Path) -> None:
         p.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
     except PermissionError:
         pass
-
-
-__all__ = [
-    "ensure_decryptor",
-    "RELEASE_VERSION",
-]

novel_downloader/core/parsers/qidian/utils/fontmap_recover.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.qidian.utils.fontmap_recover
+----------------------------------------------------------
+
+Tools for generating and applying font character mappings
+to recover obfuscated Qidian text.
+"""
+
+__all__ = [
+    "generate_font_map",
+    "apply_font_mapping",
+]
+
+import json
+import logging
+from pathlib import Path
+
+import numpy as np
+from fontTools.ttLib import TTFont
+from PIL import ImageFont
+
+logger = logging.getLogger(__name__)
+CHAR_FONT_SIZE = 52
+
+
+def generate_font_map(
+    fixed_font_path: Path,
+    random_font_path: Path,
+    char_set: set[str],
+    refl_set: set[str],
+    cache_dir: Path,
+    batch_size: int = 32,
+) -> dict[str, str]:
+    """
+    Build a mapping from scrambled font chars to real chars.
+
+    Uses OCR to compare rendered glyphs from a known (fixed) font and an
+    obfuscated (random) font. Results are cached in JSON so repeated runs
+    are faster.
+
+    :param fixed_font_path: fixed font file.
+    :param random_font_path: random font file.
+    :param char_set: Characters to match directly.
+    :param refl_set: Characters to match in flipped form.
+    :param cache_dir: Directory to save/load cached results.
+    :param batch_size: How many chars to OCR per batch.
+
+    :return: { obf_char: real_char, ... }
+    """
+    try:
+        from novel_downloader.utils.fontocr import get_font_ocr
+
+        font_ocr = get_font_ocr(batch_size=batch_size)
+    except ImportError:
+        logger.warning("[QidianParser] FontOCR not available, font decoding will skip")
+        return {}
+
+    mapping_result: dict[str, str] = {}
+    fixed_map_file = cache_dir / "fixed_font_map" / f"{Path(fixed_font_path).stem}.json"
+    fixed_map_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # load existing cache
+    try:
+        with open(fixed_map_file, encoding="utf-8") as f:
+            fixed_map = json.load(f)
+        cached_chars = set(fixed_map.keys())
+        mapping_result.update({ch: fixed_map[ch] for ch in char_set if ch in fixed_map})
+        mapping_result.update({ch: fixed_map[ch] for ch in refl_set if ch in fixed_map})
+        char_set = set(char_set) - cached_chars
+        refl_set = set(refl_set) - cached_chars
+    except Exception:
+        fixed_map = {}
+        cached_chars = set()
+
+    # prepare font renderers and cmap sets
+    try:
+        fixed_ttf = TTFont(fixed_font_path)
+        fixed_chars = {chr(c) for c in fixed_ttf.getBestCmap()}
+        fixed_font = ImageFont.truetype(str(fixed_font_path), CHAR_FONT_SIZE)
+
+        random_ttf = TTFont(random_font_path)
+        random_chars = {chr(c) for c in random_ttf.getBestCmap()}
+        random_font = ImageFont.truetype(str(random_font_path), CHAR_FONT_SIZE)
+    except Exception as e:
+        logger.error("[FontOCR] Failed to load TTF fonts: %s", e)
+        return mapping_result
+
+    def _render_batch(chars: list[tuple[str, bool]]) -> list[tuple[str, np.ndarray]]:
+        out = []
+        for ch, reflect in chars:
+            if ch in fixed_chars:
+                font = fixed_font
+            elif ch in random_chars:
+                font = random_font
+            else:
+                continue
+            img = font_ocr.render_char_image_array(ch, font, reflect)
+            if img is not None:
+                out.append((ch, img))
+        return out
+
+    # process normal and reflected sets together
+    for chars, reflect in [(list(char_set), False), (list(refl_set), True)]:
+        for batch_chars in font_ocr._chunked(chars, font_ocr._batch_size):
+            # render all images in this batch
+            to_render = [(ch, reflect) for ch in batch_chars]
+            rendered = _render_batch(to_render)
+            if not rendered:
+                continue
+
+            # query OCR+vec simultaneously
+            imgs_to_query = [img for (ch, img) in rendered]
+            fused = font_ocr.predict(imgs_to_query, top_k=1)
+
+            # pick best per char, apply threshold + cache
+            for (ch, _), preds in zip(rendered, fused, strict=False):
+                if not preds:
+                    continue
+                real_char, _ = preds[0]
+                mapping_result[ch] = real_char
+                fixed_map[ch] = real_char
+
+    # persist updated fixed_map
+    try:
+        with open(fixed_map_file, "w", encoding="utf-8") as f:
+            json.dump(fixed_map, f, ensure_ascii=False, indent=2)
+    except Exception as e:
+        logger.error("[FontOCR] Failed to save fixed map: %s", e)
+
+    return mapping_result
+
+
+def apply_font_mapping(text: str, font_map: dict[str, str]) -> str:
+    """
+    Replace each character in `text` using `font_map`,
+    leaving unmapped characters unchanged.
+
+    :param text: The input string, possibly containing obfuscated font chars.
+    :param font_map: A dict mapping obfuscated chars to real chars.
+    :return: The de-obfuscated text.
+    """
+    return "".join(font_map.get(ch, ch) for ch in text)

novel_downloader/core/parsers/qidian/utils/helpers.py

@@ -4,10 +4,6 @@ novel_downloader.core.parsers.qidian.utils.helpers
 --------------------------------------------------
 
 Shared utility functions for parsing Qidian pages.
-
-This module provides reusable helpers to:
-- Extract SSR-rendered JSON page context and structured chapter metadata.
-- Identify VIP chapters, encrypted content, and viewability conditions.
 """
 
 import json

novel_downloader/core/parsers/qidian/utils/node_decryptor.py

@@ -80,10 +80,10 @@ class QidianNodeDecryptor:
 
             # 3) Download the Fock JS module from Qidian CDN if missing
             if not self.QIDIAN_FOCK_JS_PATH.exists():
-                from novel_downloader.utils.network import download_js_file
+                from novel_downloader.utils.network import download
 
                 try:
-                    download_js_file(
+                    download(
                         self.QIDIAN_FOCK_JS_URL,
                         self.script_dir,
                         on_exist="overwrite",

novel_downloader/core/parsers/quanben5.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.quanben5
+--------------------------------------
+
+"""
+
+from datetime import datetime
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["quanben5"],
+)
+class Quanben5Parser(BaseParser):
+    """
+    Parser for 全本小说网 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+        book_name = self._first_str(tree.xpath("//h3/span/text()"))
+        author = self._first_str(
+            tree.xpath(
+                '//p[@class="info"][contains(., "作者")]/span[@class="author"]/text()'
+            )
+        )
+        cover_url = self._first_str(tree.xpath('//div[@class="pic"]/img/@src'))
+        category = self._first_str(
+            tree.xpath('//p[@class="info"][contains(., "类别")]/span/text()')
+        )
+        tags = [category] if category else []
+        update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        summary = self._first_str(tree.xpath('//p[@class="description"]/text()'))
+
+        chapters: list[ChapterInfoDict] = []
+        for li in tree.xpath('//ul[@class="list"]/li'):
+            link = li.xpath(".//a")[0]
+            href = link.get("href", "").strip()
+            title = self._first_str(link.xpath(".//span/text()"))
+            # '/n/toutian/83840.html' -> '83840'
+            chapter_id = href.rstrip(".html").split("/")[-1]
+            chapters.append({"title": title, "url": href, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # Extract the chapter title
+        title = self._first_str(tree.xpath('//h1[@class="title1"]/text()'))
+
+        # Extract all <p> text within the content container
+        paragraphs = tree.xpath('//div[@id="content"]/p/text()')
+        # Clean whitespace and join with double newlines
+        content = "\n".join(p.strip() for p in paragraphs if p.strip())
+
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "quanben5"},
+        }

novel_downloader/core/parsers/registry.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.registry
+--------------------------------------
+
+Registry and factory helpers for creating site-specific parsers.
+"""
+
+__all__ = ["register_parser", "get_parser"]
+
+from collections.abc import Callable, Sequence
+from typing import TypeVar
+
+from novel_downloader.core.interfaces import ParserProtocol
+from novel_downloader.models import ParserConfig
+
+ParserBuilder = Callable[[ParserConfig], ParserProtocol]
+
+P = TypeVar("P", bound=ParserProtocol)
+_PARSER_MAP: dict[str, ParserBuilder] = {}
+
+
+def register_parser(
+    site_keys: Sequence[str],
+) -> Callable[[type[P]], type[P]]:
+    """
+    Decorator to register a parser class under given keys.
+
+    :param site_keys: Sequence of site identifiers
+    :param backends: Sequence of backend types
+    :return: A class decorator that populates _PARSER_MAP.
+    """
+
+    def decorator(cls: type[P]) -> type[P]:
+        for site in site_keys:
+            site_lower = site.lower()
+            _PARSER_MAP[site_lower] = cls
+        return cls
+
+    return decorator
+
+
+def get_parser(site: str, config: ParserConfig) -> ParserProtocol:
+    """
+    Returns a site-specific parser instance.
+
+    :param site: Site name (e.g., 'qidian')
+    :param config: Configuration for the parser
+    :return: An instance of a parser class
+    """
+    site_key = site.lower()
+    try:
+        parser_cls = _PARSER_MAP[site_key]
+    except KeyError as err:
+        raise ValueError(f"Unsupported site: {site!r}") from err
+
+    return parser_cls(config)