novel-downloader 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/core/exporters/linovelib/txt.py

@@ -3,22 +3,19 @@
 novel_downloader.core.exporters.linovelib.txt
 ---------------------------------------------
 
-Contains the logic for exporting novel content as a single `.txt` file.
-
-This module defines `linovelib_export_as_txt` function, which assembles and formats
-a novel based on metadata and chapter files found in the raw data directory.
-It is intended to be used by `LinovelibExporter` as part of the save/export process.
+Defines `linovelib_export_as_txt` to assemble and export a Linovelib novel
+into a single `.txt` file. Intended for use by `LinovelibExporter`.
 """
 
 from __future__ import annotations
 
-import json
 from typing import TYPE_CHECKING
 
-from novel_downloader.utils.file_utils import save_as_txt
-from novel_downloader.utils.text_utils import (
-    format_chapter,
+from novel_downloader.core.exporters.txt_util import (
+    build_txt_chapter,
+    build_txt_header,
 )
+from novel_downloader.utils import get_cleaner, save_as_txt
 
 if TYPE_CHECKING:
     from .main_exporter import LinovelibExporter
@@ -29,56 +26,69 @@ def linovelib_export_as_txt(
     book_id: str,
 ) -> None:
     """
-    将 save_path 文件夹中该小说的所有章节 json 文件合并保存为一个完整的 txt 文件,
-    并保存到 out_path 下
-
-    处理流程：
-      1. 从 book_info.json 中加载书籍信息 (包含书名、作者、简介及卷章节列表)
-      2. 遍历各卷, 每个卷先追加卷标题, 然后依次追加该卷下各章节的标题和内容
-      3. 将书籍元信息 (书名、作者、原文截至、内容简介) 与所有章节内容拼接
-      4. 将最终结果保存到 out_path 下 (例如：`{book_name}.txt`)
-
-    :param book_id: Identifier of the novel (used as subdirectory name).
+    Export a novel as a single text file by merging all chapter data.
+
+    Steps:
+      1. Read metadata from `book_info.json`.
+      2. For each volume:
+         - Clean & append the volume title.
+         - Clean & append optional volume intro.
+         - Batch-fetch all chapters in this volume to minimize SQLite overhead.
+         - For each chapter: clean title & content, then append.
+      3. Build a header block with metadata.
+      4. Concatenate header + all chapter blocks, then save as `{book_name}.txt`.
+
+    :param exporter: The LinovelibExporter instance.
+    :param book_id: Identifier of the novel (subdirectory under raw data).
     """
     TAG = "[exporter]"
     # --- Paths & options ---
-    raw_base = exporter._raw_data_dir / book_id
     out_dir = exporter.output_dir
     out_dir.mkdir(parents=True, exist_ok=True)
+    cleaner = get_cleaner(
+        enabled=exporter._config.clean_text,
+        config=exporter._config.cleaner_cfg,
+    )
 
     # --- Load book_info.json ---
-    info_path = raw_base / "book_info.json"
-    try:
-        info_text = info_path.read_text(encoding="utf-8")
-        book_info = json.loads(info_text)
-    except Exception as e:
-        exporter.logger.error("%s Failed to load %s: %s", TAG, info_path, e)
+    book_info = exporter._load_book_info(book_id)
+    if not book_info:
         return
 
     # --- Compile chapters ---
     parts: list[str] = []
-    volumes = book_info.get("volumes", [])
-
-    for vol in volumes:
-        vol_name = vol.get("volume_name", "").strip()
-        vol_intro = vol.get("volume_intro", "").strip()
-        if vol_name:
-            volume_header = f"\n\n{'=' * 6} {vol_name} {'=' * 6}\n\n"
-            parts.append(volume_header)
-            exporter.logger.info("%s Processing volume: %s", TAG, vol_name)
+
+    for vol in book_info.get("volumes", []):
+        vol_title = cleaner.clean_title(vol.get("volume_name", ""))
+        if vol_title:
+            parts.append(f"\n\n{'=' * 6} {vol_title} {'=' * 6}\n\n")
+            exporter.logger.info("%s Processing volume: %s", TAG, vol_title)
+
+        vol_intro = cleaner.clean_content(vol.get("volume_intro", ""))
         if vol_intro:
             parts.append(f"{vol_intro}\n\n")
-        for chap in vol.get("chapters", []):
-            chap_id = chap.get("chapterId")
-            chap_title = chap.get("title", "")
+
+        # Batch-fetch chapters for this volume
+        chap_ids = [
+            chap.get("chapterId")
+            for chap in vol.get("chapters", [])
+            if chap.get("chapterId")
+        ]
+        chap_map = exporter._get_chapters(book_id, chap_ids)
+
+        for chap_meta in vol.get("chapters", []):
+            chap_id = chap_meta.get("chapterId")
             if not chap_id:
-                exporter.logger.warning("%s Missing chapterId, skipping: %s", TAG, chap)
+                exporter.logger.warning(
+                    "%s Missing chapterId, skipping: %s", TAG, chap_meta
+                )
                 continue
 
-            chapter_data = exporter._get_chapter(book_id, chap_id)
-            if not chapter_data:
+            chap_title = cleaner.clean_title(chap_meta.get("title", ""))
+            data = chap_map.get(chap_id)
+            if not data:
                 exporter.logger.info(
-                    "%s Missing chapter file in: %s (%s), skipping.",
+                    "%s Missing chapter: %s (%s), skipping.",
                     TAG,
                     chap_title,
                     chap_id,
@@ -86,33 +96,27 @@ def linovelib_export_as_txt(
                 continue
 
             # Extract structured fields
-            title = chapter_data.get("title", chap_title).strip()
-            content = chapter_data.get("content", "").strip()
+            title = cleaner.clean_title(data.get("title", chap_title))
+            content = cleaner.clean_content(data.get("content", ""))
 
-            parts.append(format_chapter(title, content, ""))
+            parts.append(build_txt_chapter(title=title, paragraphs=content, extras={}))
 
     # --- Build header ---
-    name = book_info.get("book_name")
-    author = book_info.get("author")
-    words = book_info.get("word_count")
-    updated = book_info.get("update_time")
-    summary = book_info.get("summary")
+    name = book_info.get("book_name") or ""
+    author = book_info.get("author") or ""
+    words = book_info.get("word_count") or ""
+    updated = book_info.get("update_time") or ""
+    summary = book_info.get("summary") or ""
 
-    fields = [
+    header_fields = [
         ("书名", name),
         ("作者", author),
         ("总字数", words),
         ("更新日期", updated),
+        ("内容简介", summary),
     ]
-    header_lines = [f"{label}: {value}" for label, value in fields if value]
-
-    if summary:
-        header_lines.append("内容简介:")
-        header_lines.append(summary)
-
-    header_lines += ["", "-" * 10, ""]
 
-    header = "\n".join(header_lines)
+    header = build_txt_header(header_fields)
 
     final_text = header + "\n\n" + "\n\n".join(parts).strip()
 
@@ -121,9 +125,9 @@ def linovelib_export_as_txt(
     out_path = out_dir / out_name
 
     # --- Save final text ---
-    try:
-        save_as_txt(content=final_text, filepath=out_path)
+    result = save_as_txt(content=final_text, filepath=out_path)
+    if result:
         exporter.logger.info("%s Novel saved to: %s", TAG, out_path)
-    except Exception as e:
-        exporter.logger.error("%s Failed to save file: %s", TAG, e)
+    else:
+        exporter.logger.error("%s Failed to write novel to %s", TAG, out_path)
     return

novel_downloader/core/exporters/qianbi.py

@@ -5,21 +5,18 @@ novel_downloader.core.exporters.qianbi
 
 """
 
+__all__ = ["QianbiExporter"]
+
+from novel_downloader.core.exporters.registry import register_exporter
 from novel_downloader.models import ExporterConfig
 
 from .common import CommonExporter
 
 
+@register_exporter(site_keys=["qianbi"])
 class QianbiExporter(CommonExporter):
     def __init__(
         self,
         config: ExporterConfig,
     ):
-        super().__init__(
-            config,
-            site="qianbi",
-            chap_folders=["chapters"],
-        )
-
-
-__all__ = ["QianbiExporter"]
+        super().__init__(config, site="qianbi")

novel_downloader/core/exporters/qidian.py

@@ -8,12 +8,25 @@ of novels sourced from Qidian (起点中文网). It implements the platform-spec
 logic required to structure and export novel content into desired formats.
 """
 
+__all__ = ["QidianExporter"]
+
+from novel_downloader.core.exporters.registry import register_exporter
 from novel_downloader.models import ExporterConfig
 
 from .common import CommonExporter
 
 
+@register_exporter(site_keys=["qidian", "qd"])
 class QidianExporter(CommonExporter):
+    """ """
+
+    DEFAULT_SOURCE_ID = 0
+    ENCRYPTED_SOURCE_ID = 1
+    PRIORITIES_MAP = {
+        DEFAULT_SOURCE_ID: 0,
+        ENCRYPTED_SOURCE_ID: 1,
+    }
+
     def __init__(
         self,
         config: ExporterConfig,
@@ -21,8 +34,5 @@ class QidianExporter(CommonExporter):
         super().__init__(
             config,
             site="qidian",
-            chap_folders=["chapters", "encrypted_chapters"],
+            priorities=self.PRIORITIES_MAP,
         )
-
-
-__all__ = ["QidianExporter"]

novel_downloader/core/exporters/registry.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.exporters.registry
+----------------------------------------
+
+"""
+
+__all__ = ["register_exporter", "get_exporter"]
+
+from collections.abc import Callable, Sequence
+from typing import TypeVar
+
+from novel_downloader.core.interfaces import ExporterProtocol
+from novel_downloader.models import ExporterConfig
+
+ExporterBuilder = Callable[[ExporterConfig], ExporterProtocol]
+
+E = TypeVar("E", bound=ExporterProtocol)
+_EXPORTER_MAP: dict[str, ExporterBuilder] = {}
+
+
+def register_exporter(
+    site_keys: Sequence[str],
+) -> Callable[[type[E]], type[E]]:
+    """
+    Decorator to register a exporter class under given keys.
+
+    :param site_keys: Sequence of site identifiers
+    :return: A class decorator that populates _EXPORTER_MAP.
+    """
+
+    def decorator(cls: type[E]) -> type[E]:
+        for key in site_keys:
+            _EXPORTER_MAP[key.lower()] = cls
+        return cls
+
+    return decorator
+
+
+def get_exporter(site: str, config: ExporterConfig) -> ExporterProtocol:
+    """
+    Returns a site-specific exporter instance.
+
+    :param site: Site name (e.g., 'qidian')
+    :param config: Configuration for the exporter
+    :return: An instance of a exporter class
+    """
+    site_key = site.lower()
+    try:
+        exporter_cls = _EXPORTER_MAP[site_key]
+    except KeyError as err:
+        raise ValueError(f"Unsupported site: {site}") from err
+    return exporter_cls(config)

novel_downloader/core/exporters/sfacg.py

@@ -5,21 +5,18 @@ novel_downloader.core.exporters.sfacg
 
 """
 
+__all__ = ["SfacgExporter"]
+
+from novel_downloader.core.exporters.registry import register_exporter
 from novel_downloader.models import ExporterConfig
 
 from .common import CommonExporter
 
 
+@register_exporter(site_keys=["sfacg"])
 class SfacgExporter(CommonExporter):
     def __init__(
         self,
         config: ExporterConfig,
     ):
-        super().__init__(
-            config,
-            site="sfacg",
-            chap_folders=["chapters"],
-        )
-
-
-__all__ = ["SfacgExporter"]
+        super().__init__(config, site="sfacg")

novel_downloader/core/exporters/txt_util.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.exporters.txt_util
+----------------------------------------
+
+Utilities for generating plain-text exports of novel content.
+"""
+
+__all__ = [
+    "build_txt_header",
+    "build_txt_chapter",
+]
+
+import re
+
+_IMG_TAG_RE = re.compile(r"<img[^>]*>")
+
+
+def build_txt_header(fields: list[tuple[str, str]]) -> str:
+    """
+    Build a simple text header from label-value pairs, followed by a dashed separator.
+
+    :param fields: List of (label, value) pairs.
+    :return: A single string containing the formatted header.
+    """
+    header_lines = [f"{label}: {value}" for label, value in fields if value]
+    header_lines += ["", "-" * 10, ""]
+    return "\n".join(header_lines)
+
+
+def build_txt_chapter(
+    title: str,
+    paragraphs: str,
+    extras: dict[str, str] | None = None,
+) -> str:
+    """
+    Build a formatted chapter text block including title, body paragraphs,
+    and optional extra sections.
+
+    - Strips any `<img...>` tags from paragraphs.
+    - Title appears first (stripped of surrounding whitespace).
+    - Each non-blank line in `paragraphs` becomes its own paragraph.
+
+    :param title:      Chapter title.
+    :param paragraphs: Raw multi-line string. Blank lines are ignored.
+    :param extras:     Optional dict mapping section titles to multi-line strings.
+    :return:           A string where title, paragraphs, and extras are joined by lines.
+    """
+    parts: list[str] = [title.strip()]
+
+    # add each nonempty paragraph line
+    paragraphs = _IMG_TAG_RE.sub("", paragraphs)
+    for ln in paragraphs.splitlines():
+        line = ln.strip()
+        if line:
+            parts.append(line)
+
+    if extras:
+        for title, text in extras.items():
+            lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+            if not lines:
+                continue
+            parts.append("---")
+            parts.append(title.strip())
+            parts.extend(lines)
+
+    return "\n\n".join(parts)

novel_downloader/core/exporters/yamibo.py

@@ -5,21 +5,18 @@ novel_downloader.core.exporters.yamibo
 
 """
 
+__all__ = ["YamiboExporter"]
+
+from novel_downloader.core.exporters.registry import register_exporter
 from novel_downloader.models import ExporterConfig
 
 from .common import CommonExporter
 
 
+@register_exporter(site_keys=["yamibo"])
 class YamiboExporter(CommonExporter):
     def __init__(
         self,
         config: ExporterConfig,
     ):
-        super().__init__(
-            config,
-            site="yamibo",
-            chap_folders=["chapters"],
-        )
-
-
-__all__ = ["YamiboExporter"]
+        super().__init__(config, site="yamibo")

novel_downloader/core/fetchers/__init__.py

@@ -16,17 +16,30 @@ Subpackages:
 - qidian (起点中文网)
 - sfacg (SF轻小说)
 - yamibo (百合会)
-- common (通用架构)
 """
 
+__all__ = [
+    "get_fetcher",
+    "BiqugeBrowser",
+    "BiqugeSession",
+    "EsjzoneBrowser",
+    "EsjzoneSession",
+    "LinovelibBrowser",
+    "LinovelibSession",
+    "QianbiBrowser",
+    "QianbiSession",
+    "QidianBrowser",
+    "QidianSession",
+    "SfacgBrowser",
+    "SfacgSession",
+    "YamiboBrowser",
+    "YamiboSession",
+]
+
 from .biquge import (
     BiqugeBrowser,
     BiqugeSession,
 )
-from .common import (
-    CommonBrowser,
-    CommonSession,
-)
 from .esjzone import (
     EsjzoneBrowser,
     EsjzoneSession,
@@ -43,6 +56,7 @@ from .qidian import (
     QidianBrowser,
     QidianSession,
 )
+from .registry import get_fetcher
 from .sfacg import (
     SfacgBrowser,
     SfacgSession,
@@ -51,22 +65,3 @@ from .yamibo import (
     YamiboBrowser,
     YamiboSession,
 )
-
-__all__ = [
-    "BiqugeBrowser",
-    "BiqugeSession",
-    "CommonBrowser",
-    "CommonSession",
-    "EsjzoneBrowser",
-    "EsjzoneSession",
-    "LinovelibBrowser",
-    "LinovelibSession",
-    "QianbiBrowser",
-    "QianbiSession",
-    "QidianBrowser",
-    "QidianSession",
-    "SfacgBrowser",
-    "SfacgSession",
-    "YamiboBrowser",
-    "YamiboSession",
-]

novel_downloader/core/fetchers/base/__init__.py

@@ -5,10 +5,10 @@ novel_downloader.core.fetchers.base
 
 """
 
-from .browser import BaseBrowser
-from .session import BaseSession
-
 __all__ = [
     "BaseBrowser",
     "BaseSession",
 ]
+
+from .browser import BaseBrowser
+from .session import BaseSession

novel_downloader/core/fetchers/base/browser.py

@@ -6,9 +6,11 @@ novel_downloader.core.fetchers.base.browser
 """
 
 import abc
+import asyncio
 import logging
 import types
-from typing import Any, Literal, Self
+from pathlib import Path
+from typing import Any, Literal, Self, TypedDict
 
 from playwright.async_api import (
     Browser,
@@ -21,7 +23,7 @@ from playwright.async_api import (
 )
 
 from novel_downloader.core.interfaces import FetcherProtocol
-from novel_downloader.models import FetcherConfig, LoginField, NewContextOptions
+from novel_downloader.models import FetcherConfig, LoginField
 from novel_downloader.utils.constants import (
     DATA_DIR,
     DEFAULT_USER_AGENT,
@@ -37,6 +39,16 @@ window.chrome = { runtime: {} };
 """.strip()
 
 
+class NewContextOptions(TypedDict, total=False):
+    user_agent: str
+    locale: str
+    storage_state: Path
+    viewport: ViewportSize
+    java_script_enabled: bool
+    ignore_https_errors: bool
+    extra_http_headers: dict[str, str]
+
+
 class BaseBrowser(FetcherProtocol, abc.ABC):
     """
     BaseBrowser wraps basic browser operations using playwright
@@ -198,12 +210,15 @@ class BaseBrowser(FetcherProtocol, abc.ABC):
         wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"]
         | None = "load",
         referer: str | None = None,
+        delay: float = 0.0,
         **kwargs: Any,
     ) -> str:
         if self._reuse_page:
-            return await self._fetch_with_reuse(url, wait_until, referer, **kwargs)
+            return await self._fetch_with_reuse(
+                url, wait_until, referer, delay, **kwargs
+            )
         else:
-            return await self._fetch_with_new(url, wait_until, referer, **kwargs)
+            return await self._fetch_with_new(url, wait_until, referer, delay, **kwargs)
 
     async def load_state(self) -> bool:
         """ """
@@ -282,11 +297,13 @@ class BaseBrowser(FetcherProtocol, abc.ABC):
         wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"]
         | None = "load",
         referer: str | None = None,
+        delay: float = 0.0,
         **kwargs: Any,
     ) -> str:
         page = await self.context.new_page()
         try:
             await page.goto(url, wait_until=wait_until, referer=referer, **kwargs)
+            await asyncio.sleep(delay)
             html: str = await page.content()
             return html
         finally:
@@ -298,11 +315,13 @@ class BaseBrowser(FetcherProtocol, abc.ABC):
         wait_until: Literal["commit", "domcontentloaded", "load", "networkidle"]
         | None = "load",
         referer: str | None = None,
+        delay: float = 0.0,
         **kwargs: Any,
     ) -> str:
         if not self._page:
             self._page = await self.context.new_page()
         await self._page.goto(url, wait_until=wait_until, referer=referer, **kwargs)
+        await asyncio.sleep(delay)
         html: str = await self._page.content()
         return html
 

novel_downloader/core/fetchers/base/session.py

@@ -21,12 +21,14 @@ from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
 
 from novel_downloader.core.interfaces import FetcherProtocol
 from novel_downloader.models import FetcherConfig, LoginField
+from novel_downloader.utils import (
+    async_sleep_with_random_delay,
+    parse_cookie_expires,
+)
 from novel_downloader.utils.constants import (
     DATA_DIR,
     DEFAULT_USER_HEADERS,
 )
-from novel_downloader.utils.cookies import parse_cookie_expires
-from novel_downloader.utils.time_utils import async_sleep_with_random_delay
 
 from .rate_limiter import TokenBucketRateLimiter
 
@@ -156,7 +158,12 @@ class BaseSession(FetcherProtocol, abc.ABC):
             await self._session.close()
         self._session = None
 
-    async def fetch(self, url: str, **kwargs: Any) -> str:
+    async def fetch(
+        self,
+        url: str,
+        encoding: str | None = None,
+        **kwargs: Any,
+    ) -> str:
         """
         Fetch the content from the given URL asynchronously, with retry support.
 
@@ -172,8 +179,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
             try:
                 async with self.session.get(url, **kwargs) as resp:
                     resp.raise_for_status()
-                    text: str = await resp.text()
-                    return text
+                    return await self._response_to_str(resp, encoding)
             except aiohttp.ClientError:
                 if attempt < self.retry_times:
                     await async_sleep_with_random_delay(
@@ -405,6 +411,25 @@ class BaseSession(FetcherProtocol, abc.ABC):
             return dict(self._session.headers)
         return self._headers.copy()
 
+    @staticmethod
+    async def _response_to_str(
+        resp: ClientResponse,
+        encoding: str | None = None,
+    ) -> str:
+        """
+        Read the full body of resp as text. First try the declared charset,
+        then on UnicodeDecodeError fall back to a lenient utf-8 decode.
+        """
+        data: bytes = await resp.read()
+        encodings = [encoding, resp.charset, "utf-8", "gb18030", "gbk"]
+        encodings_list: list[str] = [e for e in encodings if e]
+        for enc in encodings_list:
+            try:
+                return data.decode(enc)
+            except UnicodeDecodeError:
+                continue
+        return data.decode("utf-8", errors="ignore")
+
     async def __aenter__(self) -> Self:
         if self._session is None or self._session.closed:
             await self.init()

novel_downloader/core/fetchers/biquge/__init__.py

@@ -5,10 +5,10 @@ novel_downloader.core.fetchers.biquge
 
 """
 
-from .browser import BiqugeBrowser
-from .session import BiqugeSession
-
 __all__ = [
     "BiqugeBrowser",
     "BiqugeSession",
 ]
+
+from .browser import BiqugeBrowser
+from .session import BiqugeSession