novel-downloader 1.3.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/config/models.py

@@ -1,187 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.config.models
-------------------------------
-
-Defines structured configuration models using dataclasses for each
-major component in the novel_downloader pipeline.
-
-Each config section corresponds to a specific stage of the pipeline:
-- RequesterConfig: network settings for requests and DrissionPage
-- DownloaderConfig: chapter download behavior and local raw data paths
-- ParserConfig: font decoding, cache handling, and debug options
-- SaverConfig: output formatting, export formats, and filename templates
-
-These models are used to map loaded YAML or JSON config data into
-strongly typed Python objects for safer and cleaner access.
-"""
-
-from dataclasses import dataclass
-from typing import Any, Literal, TypedDict
-
-ModeType = Literal["browser", "session", "async"]
-StorageBackend = Literal["json", "sqlite"]
-
-
-# === Requesters ===
-@dataclass
-class RequesterConfig:
-    retry_times: int = 3
-    backoff_factor: float = 2.0
-    timeout: float = 30.0
-    headless: bool = True
-    user_data_folder: str = ""
-    profile_name: str = ""
-    auto_close: bool = True
-    disable_images: bool = True
-    mute_audio: bool = True
-    mode: ModeType = "session"
-    max_connections: int = 10
-    max_rps: float | None = None  # Maximum requests per second
-    username: str = ""
-    password: str = ""
-
-
-# === Downloaders ===
-@dataclass
-class DownloaderConfig:
-    request_interval: float = 5.0
-    raw_data_dir: str = "./raw_data"
-    cache_dir: str = "./novel_cache"
-    download_workers: int = 4
-    parser_workers: int = 4
-    use_process_pool: bool = False
-    skip_existing: bool = True
-    login_required: bool = False
-    save_html: bool = False
-    mode: ModeType = "session"
-    storage_backend: StorageBackend = "json"
-    storage_batch_size: int = 1
-
-
-# === Parsers ===
-@dataclass
-class ParserConfig:
-    cache_dir: str = "./novel_cache"
-    decode_font: bool = False
-    use_freq: bool = False
-    use_ocr: bool = True
-    use_vec: bool = False
-    ocr_version: str = "v1.0"
-    batch_size: int = 32
-    gpu_mem: int = 500
-    gpu_id: int | None = None
-    ocr_weight: float = 0.6
-    vec_weight: float = 0.4
-    save_font_debug: bool = False
-    mode: ModeType = "session"
-
-
-# === Savers ===
-@dataclass
-class SaverConfig:
-    cache_dir: str = "./novel_cache"
-    raw_data_dir: str = "./raw_data"
-    output_dir: str = "./downloads"
-    storage_backend: StorageBackend = "json"
-    clean_text: bool = True
-    make_txt: bool = True
-    make_epub: bool = False
-    make_md: bool = False
-    make_pdf: bool = False
-    append_timestamp: bool = True
-    filename_template: str = "{title}_{author}"
-    include_cover: bool = True
-    include_toc: bool = False
-    include_picture: bool = False
-
-
-class RuleStep(TypedDict, total=False):
-    # —— 操作类型 —— #
-    type: Literal[
-        "attr",
-        "select_one",
-        "select",
-        "find",
-        "find_all",
-        "exclude",
-        "regex",
-        "text",
-        "strip",
-        "replace",
-        "split",
-        "join",
-    ]
-
-    # —— BeautifulSoup 相关 —— #
-    selector: str | None  # CSS 选择器, 用于 select/select_one/exclude
-    name: str | None  # 标签名称, 用于 find/find_all
-    attrs: dict[str, Any] | None  # 属性过滤, 用于 find/find_all
-    limit: int | None  # find_all 的最大匹配数
-    attr: str | None  # 从元素获取属性值 (select/select_one/select_all)
-
-    # —— 正则相关 —— #
-    pattern: str | None  # 正则表达式
-    flags: int | None  # re.I, re.M 等
-    group: int | None  # 匹配结果中的第几个分组 (默认 0)
-    template: str | None  # 自定义组合, 比如 "$1$2字"
-
-    # —— 文本处理 —— #
-    chars: str | None  # strip 要去除的字符集
-    old: str | None  # replace 中要被替换的子串
-    new: str | None  # replace 中新的子串
-    count: int | None  # replace 中的最大替换次数
-    sep: str | None  # split/join 的分隔符
-    index: int | None  # split/select_all/select 之后取第几个元素
-
-
-class FieldRules(TypedDict):
-    steps: list[RuleStep]
-
-
-class ChapterFieldRules(TypedDict):
-    key: str
-    steps: list[RuleStep]
-
-
-class VolumesRulesOptional(TypedDict, total=False):
-    volume_selector: str  # 有卷时选择 volume 块的 selector
-    volume_name_steps: list[RuleStep]
-    volume_mode: str  # Optional: "normal" (default) or "mixed"
-    list_selector: str  # Optional: If "mixed" mode, parent container selector
-
-
-class VolumesRules(VolumesRulesOptional):
-    has_volume: bool  # 是否存在卷，false=未分卷
-    chapter_selector: str  # 选择 chapter 节点的 selector
-    chapter_steps: list[ChapterFieldRules]  # 提取章节信息的步骤列表
-
-
-class BookInfoRules(TypedDict, total=False):
-    book_name: FieldRules
-    author: FieldRules
-    cover_url: FieldRules
-    update_time: FieldRules
-    serial_status: FieldRules
-    word_count: FieldRules
-    summary: FieldRules
-    volumes: VolumesRules
-
-
-class ChapterRules(TypedDict, total=False):
-    title: FieldRules
-    content: FieldRules
-
-
-class SiteProfile(TypedDict):
-    book_info_url: str
-    chapter_url: str
-
-
-class SiteRules(TypedDict):
-    profile: SiteProfile
-    book_info: BookInfoRules
-    chapter: ChapterRules
-
-
-SiteRulesDict = dict[str, SiteRules]

novel_downloader/core/downloaders/base/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.base
---------------------------------------
-
-"""
-
-from .base_async import BaseAsyncDownloader
-from .base_sync import BaseDownloader
-
-__all__ = [
-    "BaseAsyncDownloader",
-    "BaseDownloader",
-]

novel_downloader/core/downloaders/base/base_async.py

@@ -1,153 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.base.base_async
--------------------------------------------------
-
-Defines the abstract base class `BaseAsyncDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
-"""
-
-import abc
-import logging
-from pathlib import Path
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.interfaces import (
-    AsyncDownloaderProtocol,
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
-    """
-    Abstract downloader that defines the initialization interface
-    and the general batch download flow.
-
-    Subclasses must implement the logic for downloading a single book.
-    """
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        self._requester = requester
-        self._parser = parser
-        self._saver = saver
-        self._config = config
-        self._site = site
-
-        self._raw_data_dir = Path(config.raw_data_dir) / site
-        self._cache_dir = Path(config.cache_dir) / site
-        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
-
-        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
-
-    async def download(self, book_ids: list[str]) -> None:
-        """
-        The general batch download process:
-          1. Iterate over all book IDs
-          2. For each ID, call `download_one()`
-
-        :param book_ids: A list of book identifiers to download.
-        """
-        await self.prepare()
-
-        # 2) batch download
-        for idx, book_id in enumerate(book_ids, start=1):
-            self.logger.debug(
-                "[%s] Starting download for %r (%s/%s)",
-                self.__class__.__name__,
-                book_id,
-                idx,
-                len(book_ids),
-            )
-            try:
-                await self.download_one(book_id)
-            except Exception as e:
-                self._handle_download_exception(book_id, e)
-
-    @abc.abstractmethod
-    async def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
-    async def prepare(self) -> None:
-        """
-        Optional hook called before downloading each book.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    @property
-    def requester(self) -> AsyncRequesterProtocol:
-        return self._requester
-
-    @property
-    def parser(self) -> ParserProtocol:
-        return self._parser
-
-    @property
-    def saver(self) -> SaverProtocol:
-        return self._saver
-
-    @property
-    def config(self) -> DownloaderConfig:
-        return self._config
-
-    @property
-    def raw_data_dir(self) -> Path:
-        return self._raw_data_dir
-
-    @property
-    def cache_dir(self) -> Path:
-        return self._cache_dir
-
-    @property
-    def site(self) -> str:
-        return self._site
-
-    @property
-    def save_html(self) -> bool:
-        return self._config.save_html
-
-    @property
-    def skip_existing(self) -> bool:
-        return self._config.skip_existing
-
-    @property
-    def login_required(self) -> bool:
-        return self._config.login_required
-
-    @property
-    def request_interval(self) -> float:
-        return self._config.request_interval
-
-    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
-        """
-        Handle download errors in a consistent way.
-
-        This method can be overridden or extended to implement retry logic, etc.
-
-        :param book_id: The ID of the book that failed.
-        :param error: The exception raised during download.
-        """
-        self.logger.warning(
-            "[%s] Failed to download %r: %s",
-            self.__class__.__name__,
-            book_id,
-            error,
-        )

novel_downloader/core/downloaders/base/base_sync.py

@@ -1,208 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.base.base_sync
-------------------------------------------------
-
-Defines the abstract base class `BaseDownloader`, which provides a
-common interface and reusable logic for all downloader implementations.
-"""
-
-import abc
-import logging
-from pathlib import Path
-
-from novel_downloader.config import DownloaderConfig
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncDownloaderProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class BaseDownloader(SyncDownloaderProtocol, abc.ABC):
-    """
-    Abstract downloader that defines the initialization interface
-    and the general batch download flow.
-
-    Subclasses must implement the logic for downloading a single book.
-    """
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-        site: str,
-    ):
-        """
-        Initialize the downloader with its components.
-
-        :param requester: Object implementing RequesterProtocol, used to fetch raw data.
-        :param parser: Object implementing ParserProtocol, used to parse page content.
-        :param saver: Object implementing SaverProtocol, used to save final output.
-        :param config: Downloader configuration object.
-        """
-        self._requester = requester
-        self._parser = parser
-        self._saver = saver
-        self._config = config
-        self._site = site
-
-        self._raw_data_dir = Path(config.raw_data_dir) / site
-        self._cache_dir = Path(config.cache_dir) / site
-        self._raw_data_dir.mkdir(parents=True, exist_ok=True)
-        self._cache_dir.mkdir(parents=True, exist_ok=True)
-
-        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
-
-    def download(self, book_ids: list[str]) -> None:
-        """
-        The general batch download process:
-          1. Iterate over all book IDs
-          2. For each ID, call `download_one()`
-
-        :param book_ids: A list of book identifiers to download.
-        """
-        self.prepare()
-
-        for idx, book_id in enumerate(book_ids, start=1):
-            self.logger.debug(
-                "[downloader] Starting download for book_id: %s (%s/%s)",
-                book_id,
-                idx,
-                len(book_ids),
-            )
-            try:
-                self.download_one(book_id)
-            except Exception as e:
-                self._handle_download_exception(book_id, e)
-
-    @abc.abstractmethod
-    def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
-    def prepare(self) -> None:
-        """
-        Optional hook called before downloading each book.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    @property
-    def requester(self) -> SyncRequesterProtocol:
-        """
-        Access the current requester.
-
-        :return: The internal requester instance.
-        """
-        return self._requester
-
-    @property
-    def parser(self) -> ParserProtocol:
-        """
-        Access the current parser.
-
-        :return: The internal parser instance.
-        """
-        return self._parser
-
-    @property
-    def saver(self) -> SaverProtocol:
-        """
-        Access the current saver.
-
-        :return: The internal saver instance.
-        """
-        return self._saver
-
-    @property
-    def config(self) -> DownloaderConfig:
-        """
-        Access the downloader configuration.
-
-        :return: The internal DownloaderConfig object.
-        """
-        return self._config
-
-    @property
-    def raw_data_dir(self) -> Path:
-        """
-        Access the root directory for storing raw downloaded data.
-
-        :return: Path to the raw data directory.
-        """
-        return self._raw_data_dir
-
-    @property
-    def cache_dir(self) -> Path:
-        """
-        Access the directory used for temporary caching during download.
-
-        :return: Path to the cache directory.
-        """
-        return self._cache_dir
-
-    @property
-    def site(self) -> str:
-        return self._site
-
-    @property
-    def save_html(self) -> bool:
-        return self._config.save_html
-
-    @property
-    def skip_existing(self) -> bool:
-        return self._config.skip_existing
-
-    @property
-    def login_required(self) -> bool:
-        return self._config.login_required
-
-    @property
-    def request_interval(self) -> float:
-        return self._config.request_interval
-
-    def set_requester(self, requester: SyncRequesterProtocol) -> None:
-        """
-        Replace the requester instance with a new one.
-
-        :param requester: The new requester to be used.
-        """
-        self._requester = requester
-
-    def set_parser(self, parser: ParserProtocol) -> None:
-        """
-        Replace the parser instance with a new one.
-
-        :param parser: The new parser to be used.
-        """
-        self._parser = parser
-
-    def set_saver(self, saver: SaverProtocol) -> None:
-        """
-        Replace the saver instance with a new one.
-
-        :param saver: The new saver to be used.
-        """
-        self._saver = saver
-
-    def _handle_download_exception(self, book_id: str, error: Exception) -> None:
-        """
-        Handle download errors in a consistent way.
-
-        This method can be overridden or extended to implement retry logic, etc.
-
-        :param book_id: The ID of the book that failed.
-        :param error: The exception raised during download.
-        """
-        self.logger.warning("[downloader] Failed to download %s: %s", book_id, error)

novel_downloader/core/downloaders/biquge/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge
-----------------------------------------
-
-"""
-
-from .biquge_async import BiqugeAsyncDownloader
-from .biquge_sync import BiqugeDownloader
-
-__all__ = [
-    "BiqugeAsyncDownloader",
-    "BiqugeDownloader",
-]

novel_downloader/core/downloaders/biquge/biquge_async.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge.biquge_async
------------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonAsyncDownloader
-from novel_downloader.core.interfaces import (
-    AsyncRequesterProtocol,
-    ParserProtocol,
-    SaverProtocol,
-)
-
-
-class BiqugeAsyncDownloader(CommonAsyncDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: AsyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/biquge/biquge_sync.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.biquge.biquge_sync
-----------------------------------------------------
-
-"""
-
-from novel_downloader.config.models import DownloaderConfig
-from novel_downloader.core.downloaders.common import CommonDownloader
-from novel_downloader.core.interfaces import (
-    ParserProtocol,
-    SaverProtocol,
-    SyncRequesterProtocol,
-)
-
-
-class BiqugeDownloader(CommonDownloader):
-    """"""
-
-    def __init__(
-        self,
-        requester: SyncRequesterProtocol,
-        parser: ParserProtocol,
-        saver: SaverProtocol,
-        config: DownloaderConfig,
-    ):
-        super().__init__(requester, parser, saver, config, "biquge")

novel_downloader/core/downloaders/common/__init__.py

@@ -1,14 +0,0 @@
-#!/usr/bin/env python3
-"""
-novel_downloader.core.downloaders.common
-----------------------------------------
-
-"""
-
-from .common_async import CommonAsyncDownloader
-from .common_sync import CommonDownloader
-
-__all__ = [
-    "CommonAsyncDownloader",
-    "CommonDownloader",
-]