novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/config/adapter.py

@@ -7,7 +7,8 @@ Defines ConfigAdapter, which maps a raw configuration dictionary and
 site name into structured dataclass-based config models.
 """
 
-from typing import Any
+import json
+from typing import Any, TypeVar, cast
 
 from novel_downloader.models import (
     BookConfig,
@@ -15,150 +16,104 @@ from novel_downloader.models import (
     ExporterConfig,
     FetcherConfig,
     ParserConfig,
+    TextCleanerConfig,
 )
-from novel_downloader.utils.constants import SUPPORTED_SITES
 
-from .site_rules import load_site_rules
+T = TypeVar("T")
 
 
 class ConfigAdapter:
     """
-    Adapter to map a raw config dict + site name into structured dataclass configs.
+    Adapter to map a raw configuration dictionary and site name
+    into structured dataclass configuration models.
     """
 
     def __init__(self, config: dict[str, Any], site: str):
         """
-        :param config: 完整加载的配置 dict
-        :param site:   当前站点名称 (e.g. "qidian")
+        Initialize the adapter.
+
+        :param config: The fully loaded configuration dictionary.
+        :param site: The current site name (e.g. "qidian").
         """
         self._config = config
         self._site = site
-
-        site_rules = load_site_rules()  # -> Dict[str, SiteRules]
-        self._supported_sites = set(site_rules.keys()) | SUPPORTED_SITES
-
-    @property
-    def site(self) -> str:
-        return self._site
-
-    @site.setter
-    def site(self, value: str) -> None:
-        self._site = value
-
-    def _get_site_cfg(self, site: str | None = None) -> dict[str, Any]:
-        """
-        获取指定站点的配置 (默认为当前适配站点)
-
-        1. 如果有 site-specific 配置, 优先返回它
-        2. 否则, 如果该站点在支持站点中, 尝试返回 'common' 配置
-        3. 否则返回空 dict
-        """
-        site = site or self._site
-        sites_cfg = self._config.get("sites", {}) or {}
-
-        if site in sites_cfg:
-            return sites_cfg[site] or {}
-
-        if site in self._supported_sites:
-            return sites_cfg.get("common", {}) or {}
-
-        return {}
+        self._site_cfg: dict[str, Any] = self._get_site_cfg()
+        self._gen_cfg: dict[str, Any] = config.get("general") or {}
 
     def get_fetcher_config(self) -> FetcherConfig:
         """
-        从 config["requests"] 中读取通用请求配置
-        返回 FetcherConfig 实例
+        Build a FetcherConfig from the raw configuration.
+
+        :return: A FetcherConfig instance with all fields populated.
         """
-        gen = self._config.get("general", {})
-        req = self._config.get("requests", {})
-        site_cfg = self._get_site_cfg()
         return FetcherConfig(
-            request_interval=gen.get("request_interval", 2.0),
-            retry_times=req.get("retry_times", 3),
-            backoff_factor=req.get("backoff_factor", 2.0),
-            timeout=req.get("timeout", 30.0),
-            max_connections=req.get("max_connections", 10),
-            max_rps=req.get("max_rps", None),
-            headless=req.get("headless", False),
-            disable_images=req.get("disable_images", False),
-            mode=site_cfg.get("mode", "session"),
-            proxy=req.get("proxy", None),
-            user_agent=req.get("user_agent", None),
-            headers=req.get("headers", None),
-            browser_type=req.get("browser_type", "chromium"),
-            verify_ssl=req.get("verify_ssl", True),
+            request_interval=self._get_gen_cfg("request_interval", 2.0),
+            retry_times=self._get_gen_cfg("retry_times", 3),
+            backoff_factor=self._get_gen_cfg("backoff_factor", 2.0),
+            timeout=self._get_gen_cfg("timeout", 30.0),
+            max_connections=self._get_gen_cfg("max_connections", 10),
+            max_rps=self._get_gen_cfg("max_rps", 1000.0),
+            user_agent=self._get_gen_cfg("user_agent", None),
+            headers=self._get_gen_cfg("headers", None),
+            verify_ssl=self._get_gen_cfg("verify_ssl", True),
+            locale_style=self._get_gen_cfg("locale_style", "simplified"),
         )
 
     def get_downloader_config(self) -> DownloaderConfig:
         """
-        从 config["general"] 和 config["sites"][site] 中读取下载器相关配置,
-        返回 DownloaderConfig 实例
+        Build a DownloaderConfig using both general and site-specific settings.
+
+        :return: A DownloaderConfig instance with all fields populated.
         """
         gen = self._config.get("general", {})
-        req = self._config.get("requests", {})
         debug = gen.get("debug", {})
-        site_cfg = self._get_site_cfg()
         return DownloaderConfig(
-            request_interval=gen.get("request_interval", 2.0),
-            retry_times=req.get("retry_times", 3),
-            backoff_factor=req.get("backoff_factor", 2.0),
+            request_interval=self._get_gen_cfg("request_interval", 2.0),
+            retry_times=self._get_gen_cfg("retry_times", 3),
+            backoff_factor=self._get_gen_cfg("backoff_factor", 2.0),
+            workers=self._get_gen_cfg("workers", 2),
+            skip_existing=self._get_gen_cfg("skip_existing", True),
+            login_required=self._site_cfg.get("login_required", False),
+            save_html=debug.get("save_html", False),
             raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
             cache_dir=gen.get("cache_dir", "./novel_cache"),
-            download_workers=gen.get("download_workers", 2),
-            parser_workers=gen.get("parser_workers", 2),
-            skip_existing=gen.get("skip_existing", True),
-            login_required=site_cfg.get("login_required", False),
-            save_html=debug.get("save_html", False),
-            mode=site_cfg.get("mode", "session"),
-            storage_backend=gen.get("storage_backend", "json"),
             storage_batch_size=gen.get("storage_batch_size", 1),
-            username=site_cfg.get("username", ""),
-            password=site_cfg.get("password", ""),
-            cookies=site_cfg.get("cookies", ""),
         )
 
     def get_parser_config(self) -> ParserConfig:
         """
-        从 config["general"]["cache_dir"]、config["general"]["debug"] 与
-        config["sites"][site] 中读取解析器相关配置, 返回 ParserConfig 实例
+        Build a ParserConfig from general, OCR, and site-specific settings.
+
+        :return: A ParserConfig instance with all fields populated.
         """
         gen = self._config.get("general", {})
         font_ocr = gen.get("font_ocr", {})
-        site_cfg = self._get_site_cfg()
         return ParserConfig(
             cache_dir=gen.get("cache_dir", "./novel_cache"),
-            use_truncation=site_cfg.get("use_truncation", True),
+            use_truncation=self._site_cfg.get("use_truncation", True),
             decode_font=font_ocr.get("decode_font", False),
-            use_freq=font_ocr.get("use_freq", False),
-            use_ocr=font_ocr.get("use_ocr", True),
-            use_vec=font_ocr.get("use_vec", False),
-            ocr_version=font_ocr.get("ocr_version", "v1.0"),
             save_font_debug=font_ocr.get("save_font_debug", False),
             batch_size=font_ocr.get("batch_size", 32),
-            gpu_mem=font_ocr.get("gpu_mem", 500),
-            gpu_id=font_ocr.get("gpu_id", None),
-            ocr_weight=font_ocr.get("ocr_weight", 0.6),
-            vec_weight=font_ocr.get("vec_weight", 0.4),
-            mode=site_cfg.get("mode", "session"),
         )
 
     def get_exporter_config(self) -> ExporterConfig:
         """
-        从 config["general"] 与 config["output"] 中读取存储器相关配置,
-        返回 ExporterConfig 实例
+        Build an ExporterConfig from output and general settings.
+
+        :return: An ExporterConfig instance with all fields populated.
         """
         gen = self._config.get("general", {})
         out = self._config.get("output", {})
+        cln = self._config.get("cleaner", {})
         fmt = out.get("formats", {})
         naming = out.get("naming", {})
         epub_opts = out.get("epub", {})
-        site_cfg = self._get_site_cfg()
+        cleaner_cfg = self._dict_to_cleaner_cfg(cln)
         return ExporterConfig(
             cache_dir=gen.get("cache_dir", "./novel_cache"),
             raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
             output_dir=gen.get("output_dir", "./downloads"),
-            storage_backend=gen.get("storage_backend", "json"),
-            clean_text=out.get("clean_text", True),
+            clean_text=cln.get("clean_text", True),
             make_txt=fmt.get("make_txt", True),
             make_epub=fmt.get("make_epub", False),
             make_md=fmt.get("make_md", False),
@@ -166,14 +121,37 @@ class ConfigAdapter:
             append_timestamp=naming.get("append_timestamp", True),
             filename_template=naming.get("filename_template", "{title}_{author}"),
             include_cover=epub_opts.get("include_cover", True),
-            include_toc=epub_opts.get("include_toc", False),
-            include_picture=epub_opts.get("include_picture", False),
-            split_mode=site_cfg.get("split_mode", "book"),
+            include_picture=epub_opts.get("include_picture", True),
+            split_mode=self._site_cfg.get("split_mode", "book"),
+            cleaner_cfg=cleaner_cfg,
         )
 
+    def get_login_config(self) -> dict[str, str]:
+        """
+        Return the subset of login fields present in current site config:
+            * `username`
+            * `password`
+            * `cookies`
+        """
+        out: dict[str, str] = {}
+        for key in ("username", "password", "cookies"):
+            val = self._site_cfg.get(key, "")
+            val = val.strip()
+            if val:
+                out[key] = val
+        return out
+
     def get_book_ids(self) -> list[BookConfig]:
         """
-        从 config["sites"][site]["book_ids"] 中提取目标书籍列表
+        Extract the list of target books from the site configuration.
+
+        The site config may specify book_ids as:
+          * a single string or integer
+          * a dict with book_id and optional start_id, end_id, ignore_ids
+          * a list of the above types
+
+        :return: A list of BookConfig dicts.
+        :raises ValueError: if the raw book_ids is neither a str/int, dict, nor list.
         """
         site_cfg = self._get_site_cfg()
         raw = site_cfg.get("book_ids", [])
@@ -182,7 +160,7 @@ class ConfigAdapter:
             return [{"book_id": str(raw)}]
 
         if isinstance(raw, dict):
-            return [self._dict_to_book_config(raw)]
+            return [self._dict_to_book_cfg(raw)]
 
         if not isinstance(raw, list):
             raise ValueError(
@@ -195,17 +173,68 @@ class ConfigAdapter:
                 if isinstance(item, str | int):
                     result.append({"book_id": str(item)})
                 elif isinstance(item, dict):
-                    result.append(self._dict_to_book_config(item))
+                    result.append(self._dict_to_book_cfg(item))
             except ValueError:
                 continue
 
         return result
 
+    def get_log_level(self) -> str:
+        """
+        Retrieve the logging level from [general.debug].
+
+        :return: The configured log level ("DEBUG", "INFO", "WARNING", "ERROR").
+        """
+        debug_cfg = self._config.get("general", {}).get("debug", {})
+        return debug_cfg.get("log_level") or "INFO"
+
+    @property
+    def site(self) -> str:
+        """
+        Get the current site name.
+        """
+        return self._site
+
+    @site.setter
+    def site(self, value: str) -> None:
+        """
+        Set a new site name for configuration lookups.
+
+        :param value: The new site key in config["sites"] to use.
+        """
+        self._site = value
+        self._site_cfg = self._get_site_cfg()
+
+    def _get_gen_cfg(self, key: str, default: T) -> T:
+        return self._site_cfg.get(key) or self._gen_cfg.get(key) or default
+
+    def _get_site_cfg(self) -> dict[str, Any]:
+        """
+        Retrieve the configuration for a specific site.
+
+        Lookup order:
+          1. If there is a site-specific entry under config["sites"], return that.
+          2. Otherwise, if a "common" entry exists under config["sites"], return that.
+          3. If neither is present, return an empty dict.
+
+        :param site: Optional override of the site name; defaults to self._site.
+        :return: The site-specific or common configuration dict.
+        """
+        sites_cfg = self._config.get("sites") or {}
+
+        if self._site in sites_cfg:
+            return sites_cfg[self._site] or {}
+
+        return sites_cfg.get("common") or {}
+
     @staticmethod
-    def _dict_to_book_config(data: dict[str, Any]) -> BookConfig:
+    def _dict_to_book_cfg(data: dict[str, Any]) -> BookConfig:
         """
-        Converts a dict to BookConfig with type normalization.
-        Raises ValueError if 'book_id' is missing.
+        Convert a dictionary to a BookConfig with normalized types.
+
+        :param data: A dict that must contain at least "book_id".
+        :return: A BookConfig dict with all values cast to strings or lists of strings.
+        :raises ValueError: if the "book_id" field is missing.
         """
         if "book_id" not in data:
             raise ValueError("Missing required field 'book_id'")
@@ -222,3 +251,70 @@ class ConfigAdapter:
             result["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
 
         return result
+
+    @classmethod
+    def _dict_to_cleaner_cfg(cls, cfg: dict[str, Any]) -> TextCleanerConfig:
+        """
+        Convert a nested dict of title/content rules into a TextCleanerConfig.
+
+        :param cfg: configuration dictionary
+        :return: fully constructed TextCleanerConfig
+        """
+        # Title rules
+        title_section = cfg.get("title", {})
+        title_remove = title_section.get("remove_patterns", [])
+        title_repl = title_section.get("replace", {})
+
+        title_ext = title_section.get("external", {})
+        if title_ext.get("enabled", False):
+            title_ext_rm_p = title_ext.get("remove_patterns", "")
+            title_ext_rp_p = title_ext.get("replace", "")
+
+            title_remove_ext = cls._load_str_list(title_ext_rm_p)
+            title_remove += title_remove_ext
+
+            title_repl_ext = cls._load_str_dict(title_ext_rp_p)
+            title_repl = {**title_repl, **title_repl_ext}
+
+        # Content rules
+        content_section = cfg.get("content", {})
+        content_remove = content_section.get("remove_patterns", [])
+        content_repl = content_section.get("replace", {})
+
+        content_ext = content_section.get("external", {})
+
+        if content_ext.get("enabled", False):
+            content_ext_rm_p = content_ext.get("remove_patterns", "")
+            content_ext_rp_p = content_ext.get("replace", "")
+
+            content_remove_ext = cls._load_str_list(content_ext_rm_p)
+            content_remove += content_remove_ext
+
+            content_repl_ext = cls._load_str_dict(content_ext_rp_p)
+            content_repl = {**content_repl, **content_repl_ext}
+
+        return TextCleanerConfig(
+            remove_invisible=cfg.get("remove_invisible", True),
+            title_remove_patterns=title_remove,
+            title_replacements=title_repl,
+            content_remove_patterns=content_remove,
+            content_replacements=content_repl,
+        )
+
+    @staticmethod
+    def _load_str_list(path: str) -> list[str]:
+        try:
+            with open(path, encoding="utf-8") as f:
+                parsed = json.load(f)
+            return cast(list[str], parsed)
+        except Exception:
+            return []
+
+    @staticmethod
+    def _load_str_dict(path: str) -> dict[str, str]:
+        try:
+            with open(path, encoding="utf-8") as f:
+                parsed = json.load(f)
+            return cast(dict[str, str], parsed)
+        except Exception:
+            return {}

novel_downloader/config/{loader.py → file_io.py}

@@ -1,24 +1,23 @@
 #!/usr/bin/env python3
 """
-novel_downloader.config.loader
-------------------------------
+novel_downloader.config.file_io
+-------------------------------
 
-Provides functionality to load Toml configuration files into Python
-dictionaries, with robust error handling and fallback support.
+Provides functionality to load Toml configuration files into Python dict
 """
 
 import json
 import logging
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 
-from novel_downloader.utils.cache import cached_load_config
 from novel_downloader.utils.constants import SETTING_FILE
 
+T = TypeVar("T")
 logger = logging.getLogger(__name__)
 
 
-def resolve_file_path(
+def _resolve_file_path(
     user_path: str | Path | None,
     local_filename: str | list[str],
     fallback_path: Path,
@@ -27,9 +26,9 @@ def resolve_file_path(
     Resolve the file path to use based on a prioritized lookup order.
 
     Priority:
-        1. A user-specified path (if provided and exists)
-        2. A file in the current working directory with the given name
-        3. A globally registered fallback path
+      1. A user-specified path (if provided and exists)
+      2. A file in the current working directory with the given name
+      3. A globally registered fallback path
 
     :param user_path: Optional user-specified file path.
     :param local_filename: File name to check in the current working directory.
@@ -115,7 +114,6 @@ def _load_by_extension(path: Path) -> dict[str, Any]:
         raise ValueError(f"Unsupported config file extension: {ext}")
 
 
-@cached_load_config
 def load_config(
     config_path: str | Path | None = None,
 ) -> dict[str, Any]:
@@ -123,9 +121,9 @@ def load_config(
     Load configuration data from a Toml file.
 
     :param config_path: Optional path to the Toml configuration file.
-    :return:            Parsed configuration as a dict.
+    :return: Parsed configuration as a dict.
     """
-    path = resolve_file_path(
+    path = _resolve_file_path(
         user_path=config_path,
         local_filename=[
             "settings.toml",
@@ -146,6 +144,46 @@ def load_config(
     return {}
 
 
+def get_config_value(keys: list[str], default: T) -> T:
+    """
+    Safely retrieve a nested config value.
+    """
+    cur = load_config()
+    for i, k in enumerate(keys):
+        if not isinstance(cur, dict):
+            return default
+        if i == len(keys) - 1:
+            val = cur.get(k, default)
+            return val if isinstance(val, type(default)) else default
+        cur = cur.get(k, {})
+    return default
+
+
+def save_config(
+    config: dict[str, Any],
+    output_path: str | Path = SETTING_FILE,
+) -> None:
+    """
+    Save configuration data to disk in JSON format.
+
+    :param config: Dictionary containing configuration data to save.
+    :param output_path: Destination path to save the config (default: SETTING_FILE).
+    :raises Exception: If writing to the file fails.
+    """
+    output = Path(output_path).expanduser().resolve()
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        with output.open("w", encoding="utf-8") as f:
+            json.dump(config, f, indent=2, ensure_ascii=False)
+    except Exception as e:
+        logger.error("[config] Failed to write config JSON '%s': %s", output, e)
+        raise
+
+    logger.info("[config] Configuration successfully saved to JSON: %s", output)
+    return
+
+
 def save_config_file(
     source_path: str | Path,
     output_path: str | Path = SETTING_FILE,
@@ -156,9 +194,9 @@ def save_config_file(
 
     :param source_path: The user-provided TOML file path.
     :param output_path: Destination path to save the config (default: SETTING_FILE).
+    :raises Exception: If writing to the file fails.
     """
     source = Path(source_path).expanduser().resolve()
-    output = Path(output_path).expanduser().resolve()
 
     if not source.is_file():
         raise FileNotFoundError(f"Source file not found: {source}")
@@ -169,17 +207,5 @@ def save_config_file(
         logger.error("[config] Failed to load config file: %s", e)
         raise ValueError(f"Invalid config file: {source}") from e
 
-    output.parent.mkdir(parents=True, exist_ok=True)
-
-    try:
-        with output.open("w", encoding="utf-8") as f:
-            json.dump(data, f, indent=2, ensure_ascii=False)
-    except Exception as e:
-        logger.error("[config] Failed to write config JSON '%s': %s", output, e)
-        raise
-
-    logger.info("[config] Configuration successfully saved to JSON: %s", output)
+    save_config(data, output_path)
     return
-
-
-__all__ = ["load_config"]

novel_downloader/core/__init__.py

@@ -12,28 +12,29 @@ downloading and processing online novel content, including:
 - Parser: Extracts structured data from HTML or SSR content.
 - Fetcher: Sends HTTP requests and manages sessions, including login if required.
 - Exporter: Responsible for exporting downloaded data into various output formats.
+- search: Provides unified search functionality across supported novel sites.
 """
 
-from .factory import (
-    get_downloader,
-    get_exporter,
-    get_fetcher,
-    get_parser,
-)
-from .interfaces import (
-    DownloaderProtocol,
-    ExporterProtocol,
-    FetcherProtocol,
-    ParserProtocol,
-)
-
 __all__ = [
     "get_downloader",
     "get_exporter",
     "get_fetcher",
     "get_parser",
+    "search",
     "DownloaderProtocol",
     "ExporterProtocol",
     "FetcherProtocol",
     "ParserProtocol",
 ]
+
+from .downloaders import get_downloader
+from .exporters import get_exporter
+from .fetchers import get_fetcher
+from .interfaces import (
+    DownloaderProtocol,
+    ExporterProtocol,
+    FetcherProtocol,
+    ParserProtocol,
+)
+from .parsers import get_parser
+from .searchers import search