novel-downloader 1.5.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/parsers/qidian/chapter_normal.py

@@ -65,17 +65,15 @@ def parse_normal_chapter(
         seq = chapter_info.get("seq", None)
         volume = chapter_info.get("extra", {}).get("volumeName", "")
 
-        chapter_text = _parse_browser_paragraph(html_str)
+        chapter_text = _parse_paragraph(
+            html_str=raw_html,
+            is_vip=vip_status(ssr_data),
+            chapter_id=chapter_id,
+            fkp=fkp,
+            fuid=parser._fuid,
+        )
         if not chapter_text:
-            chapter_text = _parse_session_paragraph(
-                html_str=raw_html,
-                is_vip=vip_status(ssr_data),
-                chapter_id=chapter_id,
-                fkp=fkp,
-                fuid=parser._fuid,
-            )
-            if not chapter_text:
-                return None
+            return None
 
         if parser._use_truncation and duplicated:
             chapter_text = truncate_half_lines(chapter_text)
@@ -103,55 +101,26 @@ def parse_normal_chapter(
     return None
 
 
-def _parse_browser_paragraph(html_str: str) -> str:
-    try:
-        tree = html.fromstring(html_str)
-        main = tree.xpath('//div[@id="app"]//div[@id="reader-content"]//main')
-        if not main:
-            return ""
-        main = main[0]
-
-        content_spans = main.xpath('.//span[contains(@class, "content-text")]')
-
-        paragraph_texts = [
-            span.text_content().strip()
-            for span in content_spans
-            if span.text_content().strip()
-        ]
-
-        chapter_text = "\n\n".join(paragraph_texts)
-        return chapter_text
-
-    except Exception as e:
-        logger.error("[Parser] _parse_paragraph failed: %s", e)
-    return ""
-
-
-def _parse_session_paragraph(
+def _parse_paragraph(
     html_str: str,
     is_vip: bool,
     chapter_id: str,
     fkp: str,
     fuid: str,
 ) -> str:
-    try:
-        raw_html = html_str
-
-        if is_vip:
-            try:
-                decryptor = get_decryptor()
-                raw_html = decryptor.decrypt(raw_html, chapter_id, fkp, fuid)
-            except Exception as e:
-                logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
-                return ""
-
-        tree = html.fromstring(raw_html)
-        paras = tree.xpath(".//p")
-        paragraph_texts = [
-            p.text_content().strip() for p in paras if p.text_content().strip()
-        ]
-        return "\n\n".join(paragraph_texts)
+    raw_html = html_str
+
+    if is_vip:
+        try:
+            decryptor = get_decryptor()
+            raw_html = decryptor.decrypt(raw_html, chapter_id, fkp, fuid)
+        except Exception as e:
+            logger.error("[Parser] decryption failed for '%s': %s", chapter_id, e)
+            return ""
 
-    except Exception as e:
-        logger.error("[Parser] _parse_paragraph failed: %s", e)
-    return ""
+    tree = html.fromstring(raw_html)
+    paras = tree.xpath(".//p")
+    paragraph_texts = [
+        p.text_content().strip() for p in paras if p.text_content().strip()
+    ]
+    return "\n".join(paragraph_texts)

novel_downloader/core/parsers/qidian/main_parser.py

@@ -10,13 +10,17 @@ from __future__ import annotations
 
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from novel_downloader.core.parsers.base import BaseParser
 from novel_downloader.core.parsers.registry import register_parser
-from novel_downloader.models import ChapterDict, ParserConfig
-from novel_downloader.utils import find_cookie_value
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ParserConfig,
+)
 from novel_downloader.utils.constants import DATA_DIR
+from novel_downloader.utils.cookies import get_cookie_value
 
 from .book_info_parser import parse_book_info
 from .chapter_router import parse_chapter
@@ -24,17 +28,13 @@ from .utils import is_encrypted
 
 logger = logging.getLogger(__name__)
 
-if TYPE_CHECKING:
-    from novel_downloader.utils.fontocr import FontOCR
-
 
 @register_parser(
     site_keys=["qidian", "qd"],
-    backends=["session", "browser"],
 )
 class QidianParser(BaseParser):
     """
-    Parser for Qidian site.
+    Parser for 起点中文网 site.
     """
 
     def __init__(
@@ -49,47 +49,20 @@ class QidianParser(BaseParser):
         """
         super().__init__(config)
 
-        # Extract and store parser flags from config
-        self._use_truncation = config.use_truncation
-        self._decode_font: bool = config.decode_font
-
         self._fixed_font_dir: Path = self._base_cache_dir / "fixed_fonts"
         self._fixed_font_dir.mkdir(parents=True, exist_ok=True)
         self._debug_dir: Path = Path.cwd() / "debug"
 
         state_files = [
-            DATA_DIR / "qidian" / "browser_state.cookies",
             DATA_DIR / "qidian" / "session_state.cookies",
         ]
-        self._fuid: str = fuid or find_cookie_value(state_files, "ywguid")
-
-        self._font_ocr: FontOCR | None = None
-        if self._decode_font:
-            try:
-                from novel_downloader.utils.fontocr import FontOCR
-            except ImportError:
-                logger.warning(
-                    "[QidianParser] FontOCR not available, font decoding will skip"
-                )
-            else:
-                self._font_ocr = FontOCR(
-                    cache_dir=self._base_cache_dir,
-                    use_freq=config.use_freq,
-                    use_ocr=config.use_ocr,
-                    use_vec=config.use_vec,
-                    batch_size=config.batch_size,
-                    gpu_mem=config.gpu_mem,
-                    gpu_id=config.gpu_id,
-                    ocr_weight=config.ocr_weight,
-                    vec_weight=config.vec_weight,
-                    font_debug=config.save_font_debug,
-                )
+        self._fuid: str = fuid or get_cookie_value(state_files, "ywguid")
 
     def parse_book_info(
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
+    ) -> BookInfoDict | None:
         """
         Parse a book info page and extract metadata and chapter structure.
 
@@ -97,7 +70,7 @@ class QidianParser(BaseParser):
         :return: Parsed metadata and chapter structure as a dictionary.
         """
         if not html_list:
-            return {}
+            return None
         return parse_book_info(html_list[0])
 
     def parse_chapter(

novel_downloader/core/parsers/qidian/utils/__init__.py

@@ -3,6 +3,7 @@
 novel_downloader.core.parsers.qidian.utils
 ------------------------------------------
 
+Utility functions and helpers for parsing and decrypting Qidian novel pages
 """
 
 __all__ = [

novel_downloader/core/parsers/qidian/utils/decryptor_fetcher.py

@@ -25,7 +25,7 @@ import requests
 from novel_downloader.utils.constants import JS_SCRIPT_DIR
 
 DEST_ROOT: Final[Path] = JS_SCRIPT_DIR
-GITHUB_OWNER: Final = "BowenZ217"
+GITHUB_OWNER: Final = "saudadez21"
 GITHUB_REPO: Final = "qidian-decryptor"
 RELEASE_VERSION: Final = "v1.0.1"
 BASE_URL: Final = f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}/releases/download/{RELEASE_VERSION}"

novel_downloader/core/parsers/qidian/utils/fontmap_recover.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.qidian.utils.fontmap_recover
+----------------------------------------------------------
+
+Tools for generating and applying font character mappings
+to recover obfuscated Qidian text.
+"""
+
+__all__ = [
+    "generate_font_map",
+    "apply_font_mapping",
+]
+
+import json
+import logging
+from pathlib import Path
+
+import numpy as np
+from fontTools.ttLib import TTFont
+from PIL import ImageFont
+
+logger = logging.getLogger(__name__)
+CHAR_FONT_SIZE = 52
+
+
+def generate_font_map(
+    fixed_font_path: Path,
+    random_font_path: Path,
+    char_set: set[str],
+    refl_set: set[str],
+    cache_dir: Path,
+    batch_size: int = 32,
+) -> dict[str, str]:
+    """
+    Build a mapping from scrambled font chars to real chars.
+
+    Uses OCR to compare rendered glyphs from a known (fixed) font and an
+    obfuscated (random) font. Results are cached in JSON so repeated runs
+    are faster.
+
+    :param fixed_font_path: fixed font file.
+    :param random_font_path: random font file.
+    :param char_set: Characters to match directly.
+    :param refl_set: Characters to match in flipped form.
+    :param cache_dir: Directory to save/load cached results.
+    :param batch_size: How many chars to OCR per batch.
+
+    :return: { obf_char: real_char, ... }
+    """
+    try:
+        from novel_downloader.utils.fontocr import get_font_ocr
+
+        font_ocr = get_font_ocr(batch_size=batch_size)
+    except ImportError:
+        logger.warning("[QidianParser] FontOCR not available, font decoding will skip")
+        return {}
+
+    mapping_result: dict[str, str] = {}
+    fixed_map_file = cache_dir / "fixed_font_map" / f"{Path(fixed_font_path).stem}.json"
+    fixed_map_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # load existing cache
+    try:
+        with open(fixed_map_file, encoding="utf-8") as f:
+            fixed_map = json.load(f)
+        cached_chars = set(fixed_map.keys())
+        mapping_result.update({ch: fixed_map[ch] for ch in char_set if ch in fixed_map})
+        mapping_result.update({ch: fixed_map[ch] for ch in refl_set if ch in fixed_map})
+        char_set = set(char_set) - cached_chars
+        refl_set = set(refl_set) - cached_chars
+    except Exception:
+        fixed_map = {}
+        cached_chars = set()
+
+    # prepare font renderers and cmap sets
+    try:
+        fixed_ttf = TTFont(fixed_font_path)
+        fixed_chars = {chr(c) for c in fixed_ttf.getBestCmap()}
+        fixed_font = ImageFont.truetype(str(fixed_font_path), CHAR_FONT_SIZE)
+
+        random_ttf = TTFont(random_font_path)
+        random_chars = {chr(c) for c in random_ttf.getBestCmap()}
+        random_font = ImageFont.truetype(str(random_font_path), CHAR_FONT_SIZE)
+    except Exception as e:
+        logger.error("[FontOCR] Failed to load TTF fonts: %s", e)
+        return mapping_result
+
+    def _render_batch(chars: list[tuple[str, bool]]) -> list[tuple[str, np.ndarray]]:
+        out = []
+        for ch, reflect in chars:
+            if ch in fixed_chars:
+                font = fixed_font
+            elif ch in random_chars:
+                font = random_font
+            else:
+                continue
+            img = font_ocr.render_char_image_array(ch, font, reflect)
+            if img is not None:
+                out.append((ch, img))
+        return out
+
+    # process normal and reflected sets together
+    for chars, reflect in [(list(char_set), False), (list(refl_set), True)]:
+        for batch_chars in font_ocr._chunked(chars, font_ocr._batch_size):
+            # render all images in this batch
+            to_render = [(ch, reflect) for ch in batch_chars]
+            rendered = _render_batch(to_render)
+            if not rendered:
+                continue
+
+            # query OCR+vec simultaneously
+            imgs_to_query = [img for (ch, img) in rendered]
+            fused = font_ocr.predict(imgs_to_query, top_k=1)
+
+            # pick best per char, apply threshold + cache
+            for (ch, _), preds in zip(rendered, fused, strict=False):
+                if not preds:
+                    continue
+                real_char, _ = preds[0]
+                mapping_result[ch] = real_char
+                fixed_map[ch] = real_char
+
+    # persist updated fixed_map
+    try:
+        with open(fixed_map_file, "w", encoding="utf-8") as f:
+            json.dump(fixed_map, f, ensure_ascii=False, indent=2)
+    except Exception as e:
+        logger.error("[FontOCR] Failed to save fixed map: %s", e)
+
+    return mapping_result
+
+
+def apply_font_mapping(text: str, font_map: dict[str, str]) -> str:
+    """
+    Replace each character in `text` using `font_map`,
+    leaving unmapped characters unchanged.
+
+    :param text: The input string, possibly containing obfuscated font chars.
+    :param font_map: A dict mapping obfuscated chars to real chars.
+    :return: The de-obfuscated text.
+    """
+    return "".join(font_map.get(ch, ch) for ch in text)

novel_downloader/core/parsers/qidian/utils/helpers.py

@@ -4,10 +4,6 @@ novel_downloader.core.parsers.qidian.utils.helpers
 --------------------------------------------------
 
 Shared utility functions for parsing Qidian pages.
-
-This module provides reusable helpers to:
-- Extract SSR-rendered JSON page context and structured chapter metadata.
-- Identify VIP chapters, encrypted content, and viewability conditions.
 """
 
 import json

novel_downloader/core/parsers/quanben5.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.quanben5
+--------------------------------------
+
+"""
+
+from datetime import datetime
+from typing import Any
+
+from lxml import html
+
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.core.parsers.registry import register_parser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+
+@register_parser(
+    site_keys=["quanben5"],
+)
+class Quanben5Parser(BaseParser):
+    """
+    Parser for 全本小说网 book pages.
+    """
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+        book_name = self._first_str(tree.xpath("//h3/span/text()"))
+        author = self._first_str(
+            tree.xpath(
+                '//p[@class="info"][contains(., "作者")]/span[@class="author"]/text()'
+            )
+        )
+        cover_url = self._first_str(tree.xpath('//div[@class="pic"]/img/@src'))
+        category = self._first_str(
+            tree.xpath('//p[@class="info"][contains(., "类别")]/span/text()')
+        )
+        tags = [category] if category else []
+        update_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        summary = self._first_str(tree.xpath('//p[@class="description"]/text()'))
+
+        chapters: list[ChapterInfoDict] = []
+        for li in tree.xpath('//ul[@class="list"]/li'):
+            link = li.xpath(".//a")[0]
+            href = link.get("href", "").strip()
+            title = self._first_str(link.xpath(".//span/text()"))
+            # '/n/toutian/83840.html' -> '83840'
+            chapter_id = href.rstrip(".html").split("/")[-1]
+            chapters.append({"title": title, "url": href, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "tags": tags,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # Extract the chapter title
+        title = self._first_str(tree.xpath('//h1[@class="title1"]/text()'))
+
+        # Extract all <p> text within the content container
+        paragraphs = tree.xpath('//div[@id="content"]/p/text()')
+        # Clean whitespace and join with double newlines
+        content = "\n".join(p.strip() for p in paragraphs if p.strip())
+
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title,
+            "content": content,
+            "extra": {"site": "quanben5"},
+        }

novel_downloader/core/parsers/registry.py

@@ -3,6 +3,7 @@
 novel_downloader.core.parsers.registry
 --------------------------------------
 
+Registry and factory helpers for creating site-specific parsers.
 """
 
 __all__ = ["register_parser", "get_parser"]
@@ -16,27 +17,24 @@ from novel_downloader.models import ParserConfig
 ParserBuilder = Callable[[ParserConfig], ParserProtocol]
 
 P = TypeVar("P", bound=ParserProtocol)
-_PARSER_MAP: dict[str, dict[str, ParserBuilder]] = {}
+_PARSER_MAP: dict[str, ParserBuilder] = {}
 
 
 def register_parser(
     site_keys: Sequence[str],
-    backends: Sequence[str],
 ) -> Callable[[type[P]], type[P]]:
     """
     Decorator to register a parser class under given keys.
 
     :param site_keys: Sequence of site identifiers
-    :param backends:  Sequence of backend types
+    :param backends: Sequence of backend types
     :return: A class decorator that populates _PARSER_MAP.
     """
 
     def decorator(cls: type[P]) -> type[P]:
         for site in site_keys:
             site_lower = site.lower()
-            bucket = _PARSER_MAP.setdefault(site_lower, {})
-            for backend in backends:
-                bucket[backend] = cls
+            _PARSER_MAP[site_lower] = cls
         return cls
 
     return decorator
@@ -52,17 +50,8 @@ def get_parser(site: str, config: ParserConfig) -> ParserProtocol:
     """
     site_key = site.lower()
     try:
-        backend_map = _PARSER_MAP[site_key]
+        parser_cls = _PARSER_MAP[site_key]
     except KeyError as err:
         raise ValueError(f"Unsupported site: {site!r}") from err
 
-    mode = config.mode
-    try:
-        parser_cls = backend_map[mode]
-    except KeyError as err:
-        raise ValueError(
-            f"Unsupported parser mode {mode!r} for site {site!r}. "
-            f"Available modes: {list(backend_map)}"
-        ) from err
-
     return parser_cls(config)

novel_downloader/core/parsers/sfacg.py

@@ -11,22 +11,32 @@ from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
 from novel_downloader.core.parsers.registry import register_parser
-from novel_downloader.models import ChapterDict
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
 
 
 @register_parser(
     site_keys=["sfacg"],
-    backends=["session", "browser"],
 )
 class SfacgParser(BaseParser):
-    """ """
+    """
+    Parser for sfacg book pages.
+    """
 
     # Book info XPaths
     _BOOK_NAME_XPATH = '//ul[@class="book_info"]//span[@class="book_newtitle"]/text()'
     _AUTHOR_INFO_XPATH = '//ul[@class="book_info"]//span[@class="book_info3"]/text()'
     _UPDATE_TIME_XPATH = '//ul[@class="book_info"]//span[@class="book_info3"]/br/following-sibling::text()'  # noqa: E501
     _COVER_URL_XPATH = '//ul[@class="book_info"]//li/img/@src'
-    _STATUS_XPATH = '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+    # _STATUS_XPATH = '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+    _STATUS_XPATH = (
+        '//ul[@class="book_info"]//div[@class="book_info2"]/span/text()'
+        ' and (contains(., "完结") or contains(., "连载"))]/text()'
+    )
     _SUMMARY_XPATH = '//ul[@class="book_profile"]/li[@class="book_bk_qs1"]/text()'
 
     # Catalog XPaths
@@ -47,54 +57,35 @@ class SfacgParser(BaseParser):
         self,
         html_list: list[str],
         **kwargs: Any,
-    ) -> dict[str, Any]:
-        """
-        Parse a book info page and extract metadata and chapter structure.
-
-        :param html_list: Raw HTML of the book info page.
-        :return: Parsed metadata and chapter structure as a dictionary.
-        """
+    ) -> BookInfoDict | None:
         if len(html_list) < 2:
-            return {}
+            return None
 
         info_tree = html.fromstring(html_list[0])
         catalog_tree = html.fromstring(html_list[1])
 
-        result: dict[str, Any] = {}
-
         # Book metadata
-        book_name = info_tree.xpath(self._BOOK_NAME_XPATH)
-        result["book_name"] = book_name[0].strip() if book_name else ""
+        book_name = self._first_str(info_tree.xpath(self._BOOK_NAME_XPATH))
 
-        book_info3 = info_tree.xpath(self._AUTHOR_INFO_XPATH)
-        result["author"] = book_info3[0].split("/")[0].strip() if book_info3 else ""
-        result["word_count"] = (
-            book_info3[0].split("/")[1].strip()
-            if book_info3 and len(book_info3[0].split("/")) > 1
-            else ""
-        )
+        book_info3_str = self._first_str(info_tree.xpath(self._AUTHOR_INFO_XPATH))
+        author, _, word_count = (p.strip() for p in book_info3_str.partition("/"))
 
-        book_info3_br = info_tree.xpath(self._UPDATE_TIME_XPATH)
-        result["update_time"] = book_info3_br[0].strip() if book_info3_br else ""
+        update_time = self._first_str(info_tree.xpath(self._UPDATE_TIME_XPATH))
 
-        cover_url = info_tree.xpath(self._COVER_URL_XPATH)
-        result["cover_url"] = "https:" + cover_url[0] if cover_url else ""
+        cover_url = "https:" + self._first_str(info_tree.xpath(self._COVER_URL_XPATH))
 
-        serial_status = info_tree.xpath(self._STATUS_XPATH)
-        result["serial_status"] = next(
-            (s for s in serial_status if "完结" in s or "连载" in s), ""
-        )
+        serial_status = self._first_str(info_tree.xpath(self._STATUS_XPATH))
 
-        summary = info_tree.xpath(self._SUMMARY_XPATH)
-        result["summary"] = "".join(summary).strip()
+        summary_elem = info_tree.xpath(self._SUMMARY_XPATH)
+        summary = "".join(summary_elem).strip()
 
         # Chapter structure
         volume_titles = catalog_tree.xpath(self._VOLUME_TITLE_XPATH)
         volume_blocks = catalog_tree.xpath(self._VOLUME_CONTENT_XPATH)
 
-        volumes = []
+        volumes: list[VolumeInfoDict] = []
         for vol_title, vol_block in zip(volume_titles, volume_blocks, strict=False):
-            chapters = []
+            chapters: list[ChapterInfoDict] = []
             for a in vol_block.xpath(self._CHAPTER_LIST_XPATH):
                 href = a.xpath("./@href")[0] if a.xpath("./@href") else ""
                 title = "".join(a.xpath(".//li//text()")).strip()
@@ -112,9 +103,18 @@ class SfacgParser(BaseParser):
                     "chapters": chapters,
                 }
             )
-        result["volumes"] = volumes
 
-        return result
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "word_count": word_count,
+            "serial_status": serial_status,
+            "summary": summary,
+            "volumes": volumes,
+            "extra": {},
+        }
 
     def parse_chapter(
         self,
@@ -122,13 +122,6 @@ class SfacgParser(BaseParser):
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
-        """
-        Parse a single chapter page and extract clean text or simplified HTML.
-
-        :param html_list: Raw HTML of the chapter page.
-        :param chapter_id: Identifier of the chapter being parsed.
-        :return: Cleaned chapter content as plain text or minimal HTML.
-        """
         if not html_list:
             return None
         keywords = [
@@ -156,7 +149,7 @@ class SfacgParser(BaseParser):
             raw_text_parts = tree.xpath(self._CHAPTER_TEXT_XPATH)
             content_lines = [txt.strip() for txt in raw_text_parts if txt.strip()]
 
-        content = "\n\n".join(content_lines).strip()
+        content = "\n".join(content_lines).strip()
         if not content:
             return None