novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/config/adapter.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.config.adapter
+-------------------------------
+
+Defines ConfigAdapter, which maps a raw configuration dictionary and
+site name into structured dataclass-based config models.
+
+Supported mappings:
+- requests          -> RequesterConfig
+- general+site      -> DownloaderConfig
+- general+site      -> ParserConfig
+- general+output    -> SaverConfig
+- sites[site]       -> book_ids list
+"""
+
+from typing import Any, Dict, List
+
+from .models import (
+    DownloaderConfig,
+    ParserConfig,
+    RequesterConfig,
+    SaverConfig,
+)
+
+
+class ConfigAdapter:
+    """
+    Adapter to map a raw config dict + site name into structured dataclass configs.
+    """
+
+    def __init__(self, config: Dict[str, Any], site: str):
+        """
+        :param config: 完整加载的配置 dict
+        :param site:   当前站点名称 (e.g. "qidian")
+        """
+        self._config = config
+        self._site = site
+
+    def set_site(self, site: str) -> None:
+        """
+        切换当前适配的站点
+        """
+        self._site = site
+
+    def get_requester_config(self) -> RequesterConfig:
+        """
+        从 config["requests"] 中读取通用请求配置 (含 DrissionPage 设置)
+        返回 RequesterConfig 实例
+        """
+        req = self._config.get("requests", {})
+        site_cfg = self._config.get("sites", {}).get(self._site, {})
+        return RequesterConfig(
+            wait_time=req.get("wait_time", 5),
+            retry_times=req.get("retry_times", 3),
+            retry_interval=req.get("retry_interval", 5),
+            timeout=req.get("timeout", 30),
+            headless=req.get("headless", True),
+            user_data_folder=req.get("user_data_folder", "./user_data"),
+            profile_name=req.get("profile_name", "Profile_1"),
+            auto_close=req.get("auto_close", True),
+            disable_images=req.get("disable_images", True),
+            mute_audio=req.get("mute_audio", True),
+            mode=site_cfg.get("mode", "session"),
+            max_rps=site_cfg.get("max_rps", None),
+        )
+
+    def get_downloader_config(self) -> DownloaderConfig:
+        """
+        从 config["general"] 和 config["sites"][site] 中读取下载器相关配置,
+        返回 DownloaderConfig 实例
+        """
+        gen = self._config.get("general", {})
+        debug = gen.get("debug", {})
+        site_cfg = self._config.get("sites", {}).get(self._site, {})
+        return DownloaderConfig(
+            request_interval=gen.get("request_interval", 5),
+            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
+            cache_dir=gen.get("cache_dir", "./cache"),
+            download_workers=gen.get("download_workers", 4),
+            parser_workers=gen.get("parser_workers", 4),
+            use_process_pool=gen.get("use_process_pool", True),
+            skip_existing=gen.get("skip_existing", True),
+            login_required=site_cfg.get("login_required", False),
+            save_html=debug.get("save_html", False),
+            mode=site_cfg.get("mode", "session"),
+        )
+
+    def get_parser_config(self) -> ParserConfig:
+        """
+        从 config["general"]["cache_dir"]、config["general"]["debug"] 与
+        config["sites"][site] 中读取解析器相关配置, 返回 ParserConfig 实例
+        """
+        gen = self._config.get("general", {})
+        site_cfg = self._config.get("sites", {}).get(self._site, {})
+        return ParserConfig(
+            cache_dir=gen.get("cache_dir", "./cache"),
+            decode_font=site_cfg.get("decode_font", False),
+            use_freq=site_cfg.get("use_freq", False),
+            use_ocr=site_cfg.get("use_ocr", True),
+            use_vec=site_cfg.get("use_vec", False),
+            ocr_version=site_cfg.get("ocr_version", "v1.0"),
+            save_font_debug=site_cfg.get("save_font_debug", False),
+            batch_size=site_cfg.get("batch_size", 32),
+            ocr_weight=site_cfg.get("ocr_weight", 0.6),
+            vec_weight=site_cfg.get("vec_weight", 0.4),
+            mode=site_cfg.get("mode", "session"),
+        )
+
+    def get_saver_config(self) -> SaverConfig:
+        """
+        从 config["general"] 与 config["output"] 中读取存储器相关配置,
+        返回 SaverConfig 实例
+        """
+        gen = self._config.get("general", {})
+        out = self._config.get("output", {})
+        fmt = out.get("formats", {})
+        naming = out.get("naming", {})
+        epub_opts = out.get("epub", {})
+        return SaverConfig(
+            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
+            output_dir=gen.get("output_dir", "./downloads"),
+            clean_text=out.get("clean_text", True),
+            make_txt=fmt.get("make_txt", True),
+            make_epub=fmt.get("make_epub", False),
+            make_md=fmt.get("make_md", False),
+            make_pdf=fmt.get("make_pdf", False),
+            append_timestamp=naming.get("append_timestamp", True),
+            filename_template=naming.get("filename_template", "{title}_{author}"),
+            include_cover=epub_opts.get("include_cover", True),
+            include_toc=epub_opts.get("include_toc", False),
+        )
+
+    def get_book_ids(self) -> List[str]:
+        """
+        从 config["sites"][site]["book_ids"] 中提取目标书籍列表
+        """
+        site_cfg = self._config.get("sites", {}).get(self._site, {})
+        raw_ids = site_cfg.get("book_ids", [])
+
+        if isinstance(raw_ids, str):
+            return [raw_ids]
+
+        if isinstance(raw_ids, int):
+            return [str(raw_ids)]
+
+        if not isinstance(raw_ids, list):
+            raise ValueError(
+                f"book_ids must be a list or string, got {type(raw_ids).__name__}"
+            )
+
+        return [str(book_id) for book_id in raw_ids]

novel_downloader/config/loader.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.config.loader
+--------------------------------
+
+Provides functionality to load YAML configuration files into Python
+dictionaries, with robust error handling and fallback support.
+
+This is typically used to load user-supplied or internal default config files.
+"""
+
+import json
+import logging
+from importlib.abc import Traversable
+from importlib.resources import as_file
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+import yaml
+
+from novel_downloader.utils.cache import cached_load_config
+from novel_downloader.utils.constants import (
+    BASE_CONFIG_PATH,
+    SETTING_FILE,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_config_path(
+    config_path: Optional[Union[str, Path]]
+) -> Optional[Union[Path, Traversable]]:
+    """
+    Resolve which configuration file to use, in this priority order:
+
+    1. User-specified path (the `config_path` argument).
+    2. `./settings.yaml` in the current working directory.
+    3. The global settings file (`SETTING_FILE`).
+    4. The internal default (`BASE_CONFIG_PATH`).
+
+    Returns a Path to the first existing file, or None if none is found.
+    """
+    # 1. Try the user-provided path
+    if config_path:
+        path = Path(config_path).expanduser().resolve()
+        if path.is_file():
+            return path
+        logger.warning("[config] Specified config file not found: %s", path)
+
+    # 2. Try ./settings.yaml in the current working directory
+    local_path = Path.cwd() / "settings.yaml"
+    if local_path.is_file():
+        logger.debug("[config] Using local settings.yaml at %s", local_path)
+        return local_path
+
+    # 3. Try the globally registered settings file
+    if SETTING_FILE.is_file():
+        logger.debug("[config] Using global settings file at %s", SETTING_FILE)
+        return SETTING_FILE
+
+    # 4. Fallback to the internal default configuration
+    try:
+        logger.debug(
+            "[config] Falling back to internal base config at %s", BASE_CONFIG_PATH
+        )
+        return BASE_CONFIG_PATH
+    except Exception as e:
+        logger.error("[config] Failed to load internal base config: %s", e)
+        return None
+
+
+@cached_load_config
+def load_config(config_path: Optional[Union[str, Path]]) -> Dict[str, Any]:
+    """
+    Load configuration data from a YAML file.
+
+    :param config_path: Optional path to the YAML configuration file.
+    :return:            Parsed configuration as a dict.
+    """
+    path = resolve_config_path(config_path)
+    if not path or not path.is_file():
+        logger.warning("[config] No valid config file found, using empty config.")
+        return {}
+
+    with as_file(path) as real_path:
+        try:
+            content = real_path.read_text(encoding="utf-8")
+            ext = real_path.suffix.lower()
+        except Exception as e:
+            logger.error("[config] Failed to read config file '%s': %s", path, e)
+            return {}
+
+    data: Any = None
+
+    if ext == ".json":
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError as e:
+            logger.error("[config] JSON parse error in '%s': %s", path, e)
+            return {}
+    else:
+        try:
+            data = yaml.safe_load(content)
+        except yaml.YAMLError as e:
+            logger.error("[config] YAML parse error in '%s': %s", path, e)
+            return {}
+
+    if data is None:
+        return {}
+    if not isinstance(data, dict):
+        logger.warning(
+            "[config] Expected dict in config file '%s', got %s",
+            path,
+            type(data).__name__,
+        )
+        return {}
+
+    return data
+
+
+def save_config_file(
+    source_path: Union[str, Path],
+    output_path: Union[str, Path] = SETTING_FILE,
+) -> None:
+    """
+    Validate a YAML/JSON config file, load it into a dict,
+    and then dump it as JSON to the internal SETTING_FILE.
+
+    :param source_path: The user-provided YAML file path.
+    :param output_path: Destination path to save the config (default: SETTING_FILE).
+    """
+    source = Path(source_path).expanduser().resolve()
+    output = Path(output_path).expanduser().resolve()
+
+    if not source.is_file():
+        raise FileNotFoundError(f"Source file not found: {source}")
+
+    ext = source.suffix.lower()
+
+    if ext in {".yaml", ".yml"}:
+        logger.debug("[config] Loading YAML for conversion: %s", source)
+        try:
+            with source.open("r", encoding="utf-8") as f:
+                data = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            logger.error("[config] Invalid YAML format: %s", e)
+            raise ValueError(f"Invalid YAML file: {source}") from e
+
+    elif ext == ".json":
+        logger.debug("[config] Loading JSON for saving: %s", source)
+        try:
+            with source.open("r", encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            logger.error("[config] Invalid JSON format: %s", e)
+            raise ValueError(f"Invalid JSON file: {source}") from e
+
+    else:
+        raise ValueError(f"Source file must be .yaml, .yml, or .json: {source}")
+
+    if not isinstance(data, dict):
+        raise ValueError(f"Config root must be a JSON/YAML object: {source}")
+
+    output.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with output.open("w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+    except Exception as e:
+        logger.error("[config] Failed to write config JSON '%s': %s", output, e)
+        raise
+
+    logger.info("[config] Configuration successfully saved to JSON: %s", output)
+    return
+
+
+__all__ = ["load_config"]

novel_downloader/config/models.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.config.models
+------------------------------
+
+Defines structured configuration models using dataclasses for each
+major component in the novel_downloader pipeline.
+
+Each config section corresponds to a specific stage of the pipeline:
+- RequesterConfig: network settings for requests and DrissionPage
+- DownloaderConfig: chapter download behavior and local raw data paths
+- ParserConfig: font decoding, cache handling, and debug options
+- SaverConfig: output formatting, export formats, and filename templates
+
+These models are used to map loaded YAML or JSON config data into
+strongly typed Python objects for safer and cleaner access.
+"""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Literal, Optional, TypedDict
+
+
+# === Requesters ===
+@dataclass
+class RequesterConfig:
+    wait_time: int = 5
+    retry_times: int = 3
+    retry_interval: int = 5
+    timeout: int = 30
+    headless: bool = True
+    user_data_folder: str = ""
+    profile_name: str = ""
+    auto_close: bool = True
+    disable_images: bool = True
+    mute_audio: bool = True
+    mode: str = "session"  # browser / session / async
+    max_rps: Optional[float] = None  # Maximum requests per second
+
+
+# === Downloaders ===
+@dataclass
+class DownloaderConfig:
+    request_interval: int = 5
+    raw_data_dir: str = "./raw_data"
+    cache_dir: str = "./novel_cache"
+    download_workers: int = 4
+    parser_workers: int = 4
+    use_process_pool: bool = False
+    skip_existing: bool = True
+    login_required: bool = False
+    save_html: bool = False
+    mode: str = "session"  # browser / session / async
+
+
+# === Parsers ===
+@dataclass
+class ParserConfig:
+    cache_dir: str = "./novel_cache"
+    decode_font: bool = False
+    use_freq: bool = False
+    use_ocr: bool = True
+    use_vec: bool = False
+    ocr_version: str = "v1.0"
+    batch_size: int = 32
+    ocr_weight: float = 0.6
+    vec_weight: float = 0.4
+    save_font_debug: bool = False
+    mode: str = "session"  # browser / session
+
+
+# === Savers ===
+@dataclass
+class SaverConfig:
+    raw_data_dir: str = "./raw_data"
+    output_dir: str = "./downloads"
+    clean_text: bool = True
+    make_txt: bool = True
+    make_epub: bool = False
+    make_md: bool = False
+    make_pdf: bool = False
+    append_timestamp: bool = True
+    filename_template: str = "{title}_{author}"
+    include_cover: bool = True
+    include_toc: bool = False
+
+
+class RuleStep(TypedDict, total=False):
+    # —— 操作类型 —— #
+    type: Literal[
+        "attr",
+        "select_one",
+        "select",
+        "find",
+        "find_all",
+        "exclude",
+        "regex",
+        "text",
+        "strip",
+        "replace",
+        "split",
+        "join",
+    ]
+
+    # —— BeautifulSoup 相关 —— #
+    selector: Optional[str]  # CSS 选择器, 用于 select/select_one/exclude
+    name: Optional[str]  # 标签名称, 用于 find/find_all
+    attrs: Optional[Dict[str, Any]]  # 属性过滤, 用于 find/find_all
+    limit: Optional[int]  # find_all 的最大匹配数
+    attr: Optional[str]  # 从元素获取属性值 (select/select_one/select_all)
+
+    # —— 正则相关 —— #
+    pattern: Optional[str]  # 正则表达式
+    flags: Optional[int]  # re.I, re.M 等
+    group: Optional[int]  # 匹配结果中的第几个分组 (默认 0)
+    template: Optional[str]  # 自定义组合, 比如 "$1$2字"
+
+    # —— 文本处理 —— #
+    chars: Optional[str]  # strip 要去除的字符集
+    old: Optional[str]  # replace 中要被替换的子串
+    new: Optional[str]  # replace 中新的子串
+    count: Optional[int]  # replace 中的最大替换次数
+    sep: Optional[str]  # split/join 的分隔符
+    index: Optional[int]  # split/select_all/select 之后取第几个元素
+
+
+class FieldRules(TypedDict):
+    steps: List[RuleStep]
+
+
+class ChapterFieldRules(TypedDict):
+    key: str
+    steps: List[RuleStep]
+
+
+class VolumesRules(TypedDict, total=False):
+    has_volume: bool  # 是否存在卷，false=未分卷
+    volume_selector: str  # 有卷时选择 volume 块的 selector
+    chapter_selector: str  # 选择 chapter 节点的 selector
+    volume_name_steps: List[RuleStep]
+    chapter_steps: List[ChapterFieldRules]  # 提取章节信息的步骤列表
+    volume_mode: str  # Optional: "normal" (default) or "mixed"
+    list_selector: str  # Optional: If "mixed" mode, parent container selector
+
+
+class BookInfoRules(TypedDict, total=False):
+    book_name: FieldRules
+    author: FieldRules
+    cover_url: FieldRules
+    update_time: FieldRules
+    serial_status: FieldRules
+    word_count: FieldRules
+    summary: FieldRules
+    volumes: VolumesRules
+
+
+class ChapterRules(TypedDict, total=False):
+    title: FieldRules
+    content: FieldRules
+
+
+class SiteProfile(TypedDict):
+    book_info_url: str
+    chapter_url: str
+
+
+class SiteRules(TypedDict):
+    profile: SiteProfile
+    book_info: BookInfoRules
+    chapter: ChapterRules
+
+
+SiteRulesDict = Dict[str, SiteRules]

novel_downloader/config/site_rules.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.config.site_rules
+----------------------------------
+
+Handles loading, saving, and caching of site-specific scraping rules.
+
+This module provides functionality to:
+- Load site rules from JSON, YAML, or TOML formats.
+- Save rules into a standard JSON format.
+"""
+
+import json
+import logging
+from pathlib import Path
+from typing import Union
+
+from novel_downloader.utils.cache import cached_load_config
+from novel_downloader.utils.constants import SITE_RULES_FILE
+from novel_downloader.utils.file_utils import save_as_json
+
+from .models import SiteRulesDict
+
+logger = logging.getLogger(__name__)
+
+
+def save_rules_as_json(
+    source_path: Union[str, Path], output_path: Union[str, Path] = SITE_RULES_FILE
+) -> None:
+    """
+    Load rules from source_path (toml, yaml, or json) and save as JSON.
+
+    :param source_path: Path to the source rules file
+                        (supports .toml, .yaml, .yml, .json).
+    :param output_path: Path where the JSON output will be saved.
+                        Defaults to SITE_RULES_FILE.
+    :raises FileNotFoundError: If the source_path does not exist.
+    :raises ValueError: If the source file format is not supported.
+    :raises Exception: If file loading or saving fails.
+    """
+    TAG = "[Config]"
+    source_path = Path(source_path)
+    output_path = Path(output_path)
+
+    if not source_path.exists():
+        raise FileNotFoundError(f"Source file {source_path} not found.")
+
+    suffix = source_path.suffix.lower()
+
+    logger.debug("%s Loading rules from %s (format: %s)", TAG, source_path, suffix)
+
+    try:
+        if suffix == ".toml":
+            import tomllib
+
+            with source_path.open("rb") as f:
+                rules_data = tomllib.load(f)
+        elif suffix in {".yaml", ".yml"}:
+            import yaml
+
+            with source_path.open("r", encoding="utf-8") as f:
+                rules_data = yaml.safe_load(f)
+        elif suffix == ".json":
+            with source_path.open("r", encoding="utf-8") as f:
+                rules_data = json.load(f)
+        else:
+            raise ValueError(f"Unsupported input format: {suffix}")
+
+    except Exception as e:
+        logger.error("%s Failed to load rules from %s: %s", TAG, source_path, str(e))
+        raise
+
+    logger.info("%s Saving rules to %s as JSON", TAG, output_path)
+
+    save_as_json(rules_data, output_path)
+    return
+
+
+@cached_load_config
+def load_site_rules(json_path: Union[str, Path] = SITE_RULES_FILE) -> SiteRulesDict:
+    """
+    Loads site scraping rules from a JSON file and caches the result for future access.
+
+    :param json_path: Path to the site rules JSON file. Defaults to SITE_RULES_FILE.
+    :return: A dictionary containing all site-specific scraping rules.
+    """
+    json_path = Path(json_path)
+    site_rules: SiteRulesDict = {}
+
+    if not json_path.exists():
+        return site_rules
+
+    with json_path.open("r", encoding="utf-8") as f:
+        site_rules = json.load(f)
+
+    return site_rules

novel_downloader/core/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core
+---------------------
+
+This package serves as the core layer of the novel_downloader system.
+
+It provides factory methods for constructing key components required for
+downloading and processing online novel content, including:
+
+- Downloader: Handles the full download lifecycle of a book or a batch of books.
+- Parser: Extracts structured data from HTML or SSR content.
+- Requester: Sends HTTP requests and manages sessions, including login if required.
+- Saver: Responsible for exporting downloaded data into various output formats.
+"""
+
+from .factory import get_downloader, get_parser, get_requester, get_saver
+
+__all__ = [
+    "get_downloader",
+    "get_parser",
+    "get_requester",
+    "get_saver",
+]

novel_downloader/core/downloaders/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.downloaders
+---------------------------------
+
+This subpackage contains concrete downloader implementations for
+specific novel platforms.
+
+Each downloader is responsible for orchestrating the full lifecycle
+of retrieving, parsing, and saving novel content for a given source.
+"""
+
+from .common_asynb_downloader import CommonAsyncDownloader
+from .common_downloader import CommonDownloader
+from .qidian_downloader import QidianDownloader
+
+__all__ = [
+    "CommonAsyncDownloader",
+    "CommonDownloader",
+    "QidianDownloader",
+]