novel-downloader 1.4.4__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/core/exporters/__init__.py

@@ -13,19 +13,10 @@ Currently supported platforms:
 - qidian (起点中文网)
 - sfacg (SF轻小说)
 - yamibo (百合会)
-- common (通用架构)
 """
 
-from .biquge import BiqugeExporter
-from .common import CommonExporter
-from .esjzone import EsjzoneExporter
-from .linovelib import LinovelibExporter
-from .qianbi import QianbiExporter
-from .qidian import QidianExporter
-from .sfacg import SfacgExporter
-from .yamibo import YamiboExporter
-
 __all__ = [
+    "get_exporter",
     "BiqugeExporter",
     "EsjzoneExporter",
     "LinovelibExporter",
@@ -33,5 +24,13 @@ __all__ = [
     "QidianExporter",
     "SfacgExporter",
     "YamiboExporter",
-    "CommonExporter",
 ]
+
+from .biquge import BiqugeExporter
+from .esjzone import EsjzoneExporter
+from .linovelib import LinovelibExporter
+from .qianbi import QianbiExporter
+from .qidian import QidianExporter
+from .registry import get_exporter
+from .sfacg import SfacgExporter
+from .yamibo import YamiboExporter

novel_downloader/core/exporters/base.py

@@ -9,6 +9,7 @@ content into various output formats.
 """
 
 import abc
+import json
 import logging
 import types
 from datetime import datetime
@@ -16,7 +17,8 @@ from pathlib import Path
 from typing import Any, Self
 
 from novel_downloader.core.interfaces import ExporterProtocol
-from novel_downloader.models import ExporterConfig
+from novel_downloader.models import ChapterDict, ExporterConfig
+from novel_downloader.utils import ChapterStorage
 
 
 class SafeDict(dict[str, Any]):
@@ -31,24 +33,33 @@ class BaseExporter(ExporterProtocol, abc.ABC):
     such as TXT, EPUB, Markdown, or PDF.
     """
 
+    DEFAULT_SOURCE_ID = 0
+    DEFAULT_PRIORITIES_MAP = {
+        DEFAULT_SOURCE_ID: 0,
+    }
+
     def __init__(
         self,
         config: ExporterConfig,
         site: str,
+        priorities: dict[int, int] | None = None,
     ):
         """
         Initialize the exporter with given configuration.
 
-        :param config: A ExporterConfig object that defines
-                        save paths, formats, and options.
+        :param config: Exporter configuration settings.
+        :param site: Identifier for the target website or source.
+        :param priorities: Mapping of source_id to priority value.
+                           Lower numbers indicate higher priority.
+                           E.X. {0: 10, 1: 100} means source 0 is preferred.
         """
         self._config = config
         self._site = site
+        self._priorities = priorities or self.DEFAULT_PRIORITIES_MAP
+        self._storage_cache: dict[str, ChapterStorage] = {}
 
-        self._cache_dir = Path(config.cache_dir) / site
         self._raw_data_dir = Path(config.raw_data_dir) / site
         self._output_dir = Path(config.output_dir)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
         self._output_dir.mkdir(parents=True, exist_ok=True)
 
         self.logger = logging.getLogger(f"{self.__class__.__name__}")
@@ -160,16 +171,84 @@ class BaseExporter(ExporterProtocol, abc.ABC):
 
         return f"{name}.{ext}"
 
+    @property
+    def site(self) -> str:
+        """
+        Get the site identifier.
+
+        :return: The site string.
+        """
+        return self._site
+
     @property
     def output_dir(self) -> Path:
-        """Access the output directory for saving files."""
+        """
+        Access the output directory for saving files.
+        """
         return self._output_dir
 
     @property
     def filename_template(self) -> str:
-        """Access the filename template."""
+        """
+        Access the filename template.
+        """
         return self._config.filename_template
 
+    def _get_chapter(
+        self,
+        book_id: str,
+        chap_id: str,
+    ) -> ChapterDict | None:
+        if book_id not in self._storage_cache:
+            return None
+        return self._storage_cache[book_id].get_best_chapter(chap_id)
+
+    def _get_chapters(
+        self,
+        book_id: str,
+        chap_ids: list[str],
+    ) -> dict[str, ChapterDict | None]:
+        if book_id not in self._storage_cache:
+            return {}
+        return self._storage_cache[book_id].get_best_chapters(chap_ids)
+
+    def _load_book_info(self, book_id: str) -> dict[str, Any]:
+        info_path = self._raw_data_dir / book_id / "book_info.json"
+        if not info_path.is_file():
+            self.logger.error("Missing metadata file: %s", info_path)
+            return {}
+
+        try:
+            text = info_path.read_text(encoding="utf-8")
+            data: Any = json.loads(text)
+            if not isinstance(data, dict):
+                self.logger.error(
+                    "Invalid JSON structure in %s: expected an object at the top",
+                    info_path,
+                )
+                return {}
+            return data
+        except json.JSONDecodeError as e:
+            self.logger.error("Corrupt JSON in %s: %s", info_path, e)
+        return {}
+
+    def _init_chapter_storages(self, book_id: str) -> None:
+        if book_id in self._storage_cache:
+            return
+        self._storage_cache[book_id] = ChapterStorage(
+            raw_base=self._raw_data_dir / book_id,
+            priorities=self._priorities,
+        )
+        self._storage_cache[book_id].connect()
+
+    def _close_chapter_storages(self) -> None:
+        for storage in self._storage_cache.values():
+            try:
+                storage.close()
+            except Exception as e:
+                self.logger.warning("Failed to close storage %s: %s", storage, e)
+        self._storage_cache.clear()
+
     def _on_close(self) -> None:
         """
         Hook method called at the beginning of close().
@@ -182,6 +261,7 @@ class BaseExporter(ExporterProtocol, abc.ABC):
         Shutdown and clean up the exporter.
         """
         self._on_close()
+        self._close_chapter_storages()
 
     def __enter__(self) -> Self:
         return self

novel_downloader/core/exporters/biquge.py

@@ -5,21 +5,18 @@ novel_downloader.core.exporters.biquge
 
 """
 
+__all__ = ["BiqugeExporter"]
+
+from novel_downloader.core.exporters.registry import register_exporter
 from novel_downloader.models import ExporterConfig
 
 from .common import CommonExporter
 
 
+@register_exporter(site_keys=["biquge", "bqg"])
 class BiqugeExporter(CommonExporter):
     def __init__(
         self,
         config: ExporterConfig,
     ):
-        super().__init__(
-            config,
-            site="biquge",
-            chap_folders=["chapters"],
-        )
-
-
-__all__ = ["BiqugeExporter"]
+        super().__init__(config, site="biquge")

novel_downloader/core/exporters/common/__init__.py

@@ -7,6 +7,6 @@ This module provides the `CommonExporter` class for
 handling the saving process of novels.
 """
 
-from .main_exporter import CommonExporter
-
 __all__ = ["CommonExporter"]
+
+from .main_exporter import CommonExporter

novel_downloader/core/exporters/common/epub.py

@@ -8,36 +8,29 @@ Contains the logic for exporting novel content as a single `.epub` file.
 
 from __future__ import annotations
 
-import html
-import json
-import re
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from novel_downloader.core.exporters.epub_util import (
-    Book,
+    build_epub_chapter,
+    download_cover,
+    finalize_export,
+    inline_remote_images,
+    prepare_builder,
+)
+from novel_downloader.utils import (
+    download,
+    get_cleaner,
+)
+from novel_downloader.utils.constants import DEFAULT_IMAGE_SUFFIX
+from novel_downloader.utils.epub import (
     Chapter,
-    StyleSheet,
     Volume,
 )
-from novel_downloader.utils.constants import CSS_MAIN_PATH
-from novel_downloader.utils.file_utils import sanitize_filename
-from novel_downloader.utils.network import download_image
-from novel_downloader.utils.text_utils import clean_chapter_title
 
 if TYPE_CHECKING:
     from .main_exporter import CommonExporter
 
-_IMAGE_WRAPPER = (
-    '<div class="duokan-image-single illus"><img src="../Images/{filename}" /></div>'
-)
-_IMG_TAG_PATTERN = re.compile(
-    r'<img\s+[^>]*src=[\'"]([^\'"]+)[\'"][^>]*>', re.IGNORECASE
-)
-_RAW_HTML_RE = re.compile(
-    r'^(<img\b[^>]*?\/>|<div class="duokan-image-single illus">.*?<\/div>)$', re.DOTALL
-)
-
 
 def common_export_as_epub(
     exporter: CommonExporter,
@@ -50,132 +43,137 @@ def common_export_as_epub(
       1. Load `book_info.json` for metadata.
       2. Generate introductory HTML and optionally include the cover image.
       3. Initialize the EPUB container.
-      4. Iterate through volumes and chapters, convert each to XHTML.
+      4. Iterate through volumes and chapters in volume-batches, convert each to XHTML.
       5. Assemble the spine, TOC, CSS and write out the final `.epub`.
 
-    :param saver: The saver instance, carrying config and path info.
+    :param exporter: The exporter instance, carrying config and path info.
     :param book_id: Identifier of the novel (used as subdirectory name).
     """
     TAG = "[exporter]"
     config = exporter._config
-    # --- Paths & options ---
+
     raw_base = exporter._raw_data_dir / book_id
-    img_dir = exporter._cache_dir / book_id / "images"
+    img_dir = raw_base / "images"
     out_dir = exporter.output_dir
+
     img_dir.mkdir(parents=True, exist_ok=True)
     out_dir.mkdir(parents=True, exist_ok=True)
 
+    cleaner = get_cleaner(
+        enabled=config.clean_text,
+        config=config.cleaner_cfg,
+    )
+
     # --- Load book_info.json ---
-    info_path = raw_base / "book_info.json"
-    try:
-        info_text = info_path.read_text(encoding="utf-8")
-        book_info = json.loads(info_text)
-    except Exception as e:
-        exporter.logger.error("%s Failed to load %s: %s", TAG, info_path, e)
+    book_info = exporter._load_book_info(book_id)
+    if not book_info:
         return
 
     book_name = book_info.get("book_name", book_id)
     book_author = book_info.get("author", "")
+
     exporter.logger.info(
         "%s Starting EPUB generation: %s (ID: %s)", TAG, book_name, book_id
     )
 
-    # --- Generate intro + cover ---
-    cover_path: Path | None = None
-    cover_url = book_info.get("cover_url", "")
-    if config.include_cover and cover_url:
-        cover_path = download_image(
-            cover_url,
-            raw_base,
-            target_name="cover",
-            on_exist="overwrite",
-        )
-        if not cover_path:
-            exporter.logger.warning("Failed to download cover from %s", cover_url)
+    cover_path = download_cover(
+        book_info.get("cover_url", ""),
+        raw_base,
+        config.include_cover,
+        exporter.logger,
+        TAG,
+    )
 
     # --- Initialize EPUB ---
-    book = Book(
+    book, main_css = prepare_builder(
+        site_name=exporter.site,
+        book_id=book_id,
         title=book_name,
         author=book_author,
         description=book_info.get("summary", ""),
-        cover_path=cover_path,
         subject=book_info.get("subject", []),
         serial_status=book_info.get("serial_status", ""),
         word_count=book_info.get("word_count", ""),
-        uid=f"{exporter.site}_{book_id}",
-    )
-    main_css = StyleSheet(
-        id="main_style",
-        content=CSS_MAIN_PATH.read_text(encoding="utf-8"),
-        filename="main.css",
+        cover_path=cover_path,
     )
-    book.add_stylesheet(main_css)
 
     # --- Compile chapters ---
     volumes = book_info.get("volumes", [])
-    for vol_index, vol in enumerate(volumes, start=1):
-        raw_vol_name = vol.get("volume_name", "")
-        raw_vol_name = raw_vol_name.replace(book_name, "").strip()
-        vol_name = raw_vol_name or f"Volume {vol_index}"
-        exporter.logger.info("Processing volume %d: %s", vol_index, vol_name)
+    if not volumes:
+        exporter.logger.warning("%s No volumes found in metadata.", TAG)
 
-        vol_cover_path: Path | None = None
+    for vol_index, vol in enumerate(volumes, start=1):
+        raw_name = vol.get("volume_name", "")
+        raw_name = cleaner.clean_title(raw_name.replace(book_name, ""))
+        vol_name = raw_name or f"Volume {vol_index}"
+        exporter.logger.info("%s Processing volume %d: %s", TAG, vol_index, vol_name)
+
+        # Batch-fetch chapters for this volume
+        chap_ids = [
+            chap.get("chapterId")
+            for chap in vol.get("chapters", [])
+            if chap.get("chapterId")
+        ]
+        chap_map = exporter._get_chapters(book_id, chap_ids)
+
+        vol_cover: Path | None = None
         vol_cover_url = vol.get("volume_cover", "")
         if vol_cover_url:
-            vol_cover_path = download_image(
+            vol_cover = download(
                 vol_cover_url,
                 img_dir,
                 on_exist="skip",
+                default_suffix=DEFAULT_IMAGE_SUFFIX,
             )
 
         curr_vol = Volume(
             id=f"vol_{vol_index}",
             title=vol_name,
-            intro=vol.get("volume_intro", ""),
-            cover=vol_cover_path,
+            intro=cleaner.clean_content(vol.get("volume_intro", "")),
+            cover=vol_cover,
         )
 
-        for chap in vol.get("chapters", []):
-            chap_id = chap.get("chapterId")
-            chap_title = chap.get("title", "")
+        for chap_meta in vol.get("chapters", []):
+            chap_id = chap_meta.get("chapterId")
             if not chap_id:
                 exporter.logger.warning(
                     "%s Missing chapterId, skipping: %s",
                     TAG,
-                    chap,
+                    chap_meta,
                 )
                 continue
 
-            chapter_data = exporter._get_chapter(book_id, chap_id)
-            if not chapter_data:
+            chap_title = cleaner.clean_title(chap_meta.get("title", ""))
+            data = chap_map.get(chap_id)
+            if not data:
                 exporter.logger.info(
-                    "%s Missing chapter file: %s (%s), skipping.",
+                    "%s Missing chapter: %s (%s), skipping.",
                     TAG,
                     chap_title,
                     chap_id,
                 )
                 continue
 
-            title = clean_chapter_title(chapter_data.get("title", "")) or chap_id
-            content: str = chapter_data.get("content", "")
-            content, img_paths = _inline_remote_images(content, img_dir)
-            chap_html = _txt_to_html(
-                chapter_title=title,
-                chapter_text=content,
-                extras={
-                    "作者说": chapter_data.get("author_say", ""),
-                },
+            title = cleaner.clean_title(data.get("title", chap_title)) or chap_id
+            content = cleaner.clean_content(data.get("content", ""))
+            extra = data.get("extra", {})
+            author_note = cleaner.clean_content(extra.get("author_say", ""))
+            content = inline_remote_images(book, content, img_dir)
+
+            chap_html = build_epub_chapter(
+                title=title,
+                paragraphs=content,
+                extras={"作者说": author_note},
             )
             curr_vol.add_chapter(
                 Chapter(
                     id=f"c_{chap_id}",
+                    filename=f"c{chap_id}.xhtml",
                     title=title,
                     content=chap_html,
                     css=[main_css],
                 )
             )
-            for img_path in img_paths:
-                book.add_image(img_path)
 
         book.add_volume(curr_vol)
 
@@ -185,93 +183,11 @@ def common_export_as_epub(
         author=book_info.get("author"),
         ext="epub",
     )
-    out_path = out_dir / sanitize_filename(out_name)
-
-    try:
-        book.export(out_path)
-        exporter.logger.info("%s EPUB successfully written to %s", TAG, out_path)
-    except Exception as e:
-        exporter.logger.error("%s Failed to write EPUB to %s: %s", TAG, out_path, e)
+    finalize_export(
+        book=book,
+        out_dir=out_dir,
+        filename=out_name,
+        logger=exporter.logger,
+        tag=TAG,
+    )
     return
-
-
-def _inline_remote_images(
-    content: str,
-    image_dir: str | Path,
-) -> tuple[str, list[Path]]:
-    """
-    Download every remote `<img src="...">` in `content` into `image_dir`,
-    and replace the original tag with _IMAGE_WRAPPER
-    pointing to the local filename.
-
-    :param content: HTML/text of the chapter containing <img> tags.
-    :param image_dir: Directory to save downloaded images into.
-    :return: A tuple (modified_content, list_of_downloaded_image_paths).
-    """
-    downloaded_images: list[Path] = []
-
-    def _replace(match: re.Match[str]) -> str:
-        url = match.group(1)
-        try:
-            # download_image returns a Path or None
-            local_path = download_image(
-                url,
-                image_dir,
-                target_name=None,
-                on_exist="skip",
-            )
-            if not local_path:
-                return match.group(0)
-
-            downloaded_images.append(local_path)
-            return _IMAGE_WRAPPER.format(filename=local_path.name)
-        except Exception:
-            return match.group(0)
-
-    modified_content = _IMG_TAG_PATTERN.sub(_replace, content)
-    return modified_content, downloaded_images
-
-
-def _txt_to_html(
-    chapter_title: str,
-    chapter_text: str,
-    extras: dict[str, str] | None = None,
-) -> str:
-    """
-    Convert chapter text and author note to styled HTML.
-
-    :param chapter_title: Title of the chapter.
-    :param chapter_text: Main content of the chapter.
-    :param extras: Optional dict of titles and content, e.g. {"作者说": "text"}.
-    :return: Rendered HTML as a string.
-    """
-
-    def _render_block(text: str) -> str:
-        lines = (line.strip() for line in text.splitlines() if line.strip())
-        out = []
-        for line in lines:
-            # preserve raw HTML, otherwise wrap in <p>
-            if _RAW_HTML_RE.match(line):
-                out.append(line)
-            else:
-                out.append(f"<p>{html.escape(line)}</p>")
-        return "\n".join(out)
-
-    parts = []
-    parts.append(f"<h2>{html.escape(chapter_title)}</h2>")
-    parts.append(_render_block(chapter_text))
-
-    if extras:
-        for title, note in extras.items():
-            note = note.strip()
-            if not note:
-                continue
-            parts.extend(
-                [
-                    "<hr />",
-                    f"<p>{html.escape(title)}</p>",
-                    _render_block(note),
-                ]
-            )
-
-    return "\n".join(parts)

novel_downloader/core/exporters/common/main_exporter.py

@@ -8,12 +8,7 @@ novel data. It defines the logic to compile, structure, and export novel content
 in plain text format based on the platform's metadata and chapter files.
 """
 
-from collections.abc import Mapping
-from typing import Any
-
 from novel_downloader.core.exporters.base import BaseExporter
-from novel_downloader.models import ExporterConfig
-from novel_downloader.utils.chapter_storage import ChapterStorage
 
 from .txt import common_export_as_txt
 
@@ -26,16 +21,6 @@ class CommonExporter(BaseExporter):
     and EPUB (.epub) files.
     """
 
-    def __init__(
-        self,
-        config: ExporterConfig,
-        site: str,
-        chap_folders: list[str] | None = None,
-    ):
-        super().__init__(config, site)
-        self._chapter_storage_cache: dict[str, list[ChapterStorage]] = {}
-        self._chap_folders: list[str] = chap_folders or ["chapters"]
-
     def export_as_txt(self, book_id: str) -> None:
         """
         Compile and export a complete novel as a single .txt file.
@@ -70,48 +55,3 @@ class CommonExporter(BaseExporter):
 
         self._init_chapter_storages(book_id)
         return common_export_as_epub(self, book_id)
-
-    @property
-    def site(self) -> str:
-        """
-        Get the site identifier.
-
-        :return: The site string.
-        """
-        return self._site
-
-    def _get_chapter(
-        self,
-        book_id: str,
-        chap_id: str,
-    ) -> Mapping[str, Any]:
-        for storage in self._chapter_storage_cache[book_id]:
-            data = storage.get(chap_id)
-            if data:
-                return data
-        return {}
-
-    def _init_chapter_storages(self, book_id: str) -> None:
-        if book_id in self._chapter_storage_cache:
-            return
-        raw_base = self._raw_data_dir / book_id
-        self._chapter_storage_cache[book_id] = [
-            ChapterStorage(
-                raw_base=raw_base,
-                namespace=ns,
-                backend_type=self._config.storage_backend,
-            )
-            for ns in self._chap_folders
-        ]
-
-    def _on_close(self) -> None:
-        """
-        Close all ChapterStorage connections in the cache.
-        """
-        for storages in self._chapter_storage_cache.values():
-            for storage in storages:
-                try:
-                    storage.close()
-                except Exception as e:
-                    self.logger.warning("Failed to close storage %s: %s", storage, e)
-        self._chapter_storage_cache.clear()