novel-downloader 1.5.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

novel_downloader/config/adapter.py

@@ -4,196 +4,172 @@ novel_downloader.config.adapter
 -------------------------------
 
 Defines ConfigAdapter, which maps a raw configuration dictionary and
-site name into structured dataclass-based config models.
+site into structured dataclass-based config models.
 """
 
+import contextlib
 import json
-from typing import Any, cast
+from collections.abc import Mapping
+from typing import Any, TypeVar
 
 from novel_downloader.models import (
     BookConfig,
     DownloaderConfig,
     ExporterConfig,
     FetcherConfig,
-    LogLevel,
     ParserConfig,
     TextCleanerConfig,
 )
 
+T = TypeVar("T")
+
 
 class ConfigAdapter:
     """
     Adapter to map a raw configuration dictionary and site name
     into structured dataclass configuration models.
-    """
 
-    _ALLOWED_LOG_LEVELS: tuple[LogLevel, ...] = (
-        "DEBUG",
-        "INFO",
-        "WARNING",
-        "ERROR",
-    )
+    Resolution order for each field:
+      1. ``config["sites"][<site>]`` (if present)
+      2. ``config["general"]`` (if present)
+      3. Hard-coded default passed by the caller
+    """
 
-    def __init__(self, config: dict[str, Any], site: str):
+    def __init__(self, config: Mapping[str, Any], site: str):
         """
-        Initialize the adapter.
+        Initialize the adapter with a configuration mapping and a site key.
 
-        :param config: The fully loaded configuration dictionary.
-        :param site:   The current site name (e.g. "qidian").
+        :param config: Fully loaded configuration mapping.
+        :param site: Current site key (e.g., ``"qidian"``).
         """
-        self._config = config
-        self._site = site
+        self._config: dict[str, Any] = dict(config)
+        self._site: str = site
 
     def get_fetcher_config(self) -> FetcherConfig:
         """
-        Build a FetcherConfig from the raw configuration.
-
-        Reads from:
-          - config["general"] for global defaults (e.g. request_interval)
-          - config["requests"] for HTTP-specific settings (timeouts, retries, etc.)
-          - site-specific overrides under config["sites"][site]
+        Build a :class:`novel_downloader.models.FetcherConfig` by resolving fields
+        from site-specific and general settings.
 
-        :return: A FetcherConfig instance with all fields populated.
+        :return: Fully populated configuration for the network fetcher.
         """
-        gen = self._config.get("general", {})
-        req = self._config.get("requests", {})
-        site_cfg = self._get_site_cfg()
+        s, g = self._site_cfg, self._gen_cfg
         return FetcherConfig(
-            request_interval=gen.get("request_interval", 2.0),
-            retry_times=req.get("retry_times", 3),
-            backoff_factor=req.get("backoff_factor", 2.0),
-            timeout=req.get("timeout", 30.0),
-            max_connections=req.get("max_connections", 10),
-            max_rps=req.get("max_rps", None),
-            headless=req.get("headless", False),
-            disable_images=req.get("disable_images", False),
-            mode=site_cfg.get("mode", "session"),
-            proxy=req.get("proxy", None),
-            user_agent=req.get("user_agent", None),
-            headers=req.get("headers", None),
-            browser_type=req.get("browser_type", "chromium"),
-            verify_ssl=req.get("verify_ssl", True),
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            timeout=self._pick("timeout", 30.0, s, g),
+            max_connections=self._pick("max_connections", 10, s, g),
+            max_rps=self._pick("max_rps", 1000.0, s, g),
+            user_agent=self._pick("user_agent", None, s, g),
+            headers=self._pick("headers", None, s, g),
+            verify_ssl=self._pick("verify_ssl", True, s, g),
+            locale_style=self._pick("locale_style", "simplified", s, g),
         )
 
     def get_downloader_config(self) -> DownloaderConfig:
         """
-        Build a DownloaderConfig using both general and site-specific settings.
+        Build a :class:`novel_downloader.models.DownloaderConfig` using both
+        general and site-specific settings.
 
-        Reads from:
-          - config["general"] for download directories, worker counts, etc.
-          - config["requests"] for retry and backoff settings
-          - config["general"]["debug"] for debug toggles (e.g. save_html)
-          - config["sites"][site] for login credentials and mode
-
-        :return: A DownloaderConfig instance with all fields populated.
+        :return: Fully populated configuration for the chapter/page downloader.
         """
-        gen = self._config.get("general", {})
-        req = self._config.get("requests", {})
-        debug = gen.get("debug", {})
-        site_cfg = self._get_site_cfg()
+        s, g = self._site_cfg, self._gen_cfg
+        debug = g.get("debug") or {}
         return DownloaderConfig(
-            request_interval=gen.get("request_interval", 2.0),
-            retry_times=req.get("retry_times", 3),
-            backoff_factor=req.get("backoff_factor", 2.0),
-            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            workers=gen.get("workers", 2),
-            skip_existing=gen.get("skip_existing", True),
-            login_required=site_cfg.get("login_required", False),
-            save_html=debug.get("save_html", False),
-            mode=site_cfg.get("mode", "session"),
-            storage_batch_size=gen.get("storage_batch_size", 1),
-            username=site_cfg.get("username", ""),
-            password=site_cfg.get("password", ""),
-            cookies=site_cfg.get("cookies", ""),
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            workers=self._pick("workers", 2, s, g),
+            skip_existing=self._pick("skip_existing", True, s, g),
+            login_required=bool(s.get("login_required", False)),
+            save_html=bool(debug.get("save_html", False)),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            storage_batch_size=g.get("storage_batch_size", 1),
         )
 
     def get_parser_config(self) -> ParserConfig:
         """
-        Build a ParserConfig from general, OCR, and site-specific settings.
-
-        Reads from:
-          - config["general"]["cache_dir"] for where to cache intermediate parses
-          - config["general"]["font_ocr"] for font-decoding and OCR options
-          - config["sites"][site] for parsing mode and truncation behavior
+        Build a :class:`novel_downloader.models.ParserConfig` from general,
+        OCR-related, and site-specific settings.
 
-        :return: A ParserConfig instance with all fields populated.
+        :return: Fully populated configuration for the parser stage.
         """
-        gen = self._config.get("general", {})
-        font_ocr = gen.get("font_ocr", {})
-        site_cfg = self._get_site_cfg()
+        g = self._gen_cfg
+        s = self._site_cfg
+        font_ocr = g.get("font_ocr") or {}
         return ParserConfig(
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            use_truncation=site_cfg.get("use_truncation", True),
-            decode_font=font_ocr.get("decode_font", False),
-            use_freq=font_ocr.get("use_freq", False),
-            use_ocr=font_ocr.get("use_ocr", True),
-            use_vec=font_ocr.get("use_vec", False),
-            ocr_version=font_ocr.get("ocr_version", "v1.0"),
-            save_font_debug=font_ocr.get("save_font_debug", False),
-            batch_size=font_ocr.get("batch_size", 32),
-            gpu_mem=font_ocr.get("gpu_mem", 500),
-            gpu_id=font_ocr.get("gpu_id", None),
-            ocr_weight=font_ocr.get("ocr_weight", 0.6),
-            vec_weight=font_ocr.get("vec_weight", 0.4),
-            mode=site_cfg.get("mode", "session"),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            use_truncation=bool(s.get("use_truncation", True)),
+            decode_font=bool(font_ocr.get("decode_font", False)),
+            save_font_debug=bool(font_ocr.get("save_font_debug", False)),
+            batch_size=int(font_ocr.get("batch_size", 32)),
         )
 
     def get_exporter_config(self) -> ExporterConfig:
         """
-        Build an ExporterConfig from output and general settings.
-
-        Reads from:
-          - config["general"] for cache and raw data directories
-          - config["output"]["formats"] for which formats to generate
-          - config["output"]["naming"] for filename templates
-          - config["output"]["epub"] for EPUB-specific options
-          - config["sites"][site] for export split mode
+        Build an :class:`novel_downloader.models.ExporterConfig` from the
+        ``output`` and ``cleaner`` sections plus general settings.
 
-        :return: An ExporterConfig instance with all fields populated.
+        :return: Fully populated configuration for text/ebook export.
         """
-        gen = self._config.get("general", {})
-        out = self._config.get("output", {})
-        cln = self._config.get("cleaner", {})
-        fmt = out.get("formats", {})
-        naming = out.get("naming", {})
-        epub_opts = out.get("epub", {})
-        site_cfg = self._get_site_cfg()
+        g = self._gen_cfg
+        out = self._config.get("output") or {}
+        cln = self._config.get("cleaner") or {}
+        fmt = out.get("formats") or {}
+        naming = out.get("naming") or {}
+        epub_opts = out.get("epub") or {}
+
         cleaner_cfg = self._dict_to_cleaner_cfg(cln)
         return ExporterConfig(
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
-            output_dir=gen.get("output_dir", "./downloads"),
-            clean_text=out.get("clean_text", True),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            output_dir=g.get("output_dir", "./downloads"),
+            clean_text=cln.get("clean_text", False),
             make_txt=fmt.get("make_txt", True),
-            make_epub=fmt.get("make_epub", False),
+            make_epub=fmt.get("make_epub", True),
             make_md=fmt.get("make_md", False),
             make_pdf=fmt.get("make_pdf", False),
             append_timestamp=naming.get("append_timestamp", True),
             filename_template=naming.get("filename_template", "{title}_{author}"),
             include_cover=epub_opts.get("include_cover", True),
-            include_toc=epub_opts.get("include_toc", False),
-            include_picture=epub_opts.get("include_picture", False),
-            split_mode=site_cfg.get("split_mode", "book"),
+            include_picture=epub_opts.get("include_picture", True),
+            split_mode=self._site_cfg.get("split_mode", "book"),
             cleaner_cfg=cleaner_cfg,
         )
 
+    def get_login_config(self) -> dict[str, str]:
+        """
+        Extract login-related fields from the current site configuration.
+        Only non-empty string values are returned; values are stripped.
+
+        :return: A subset of ``{"username","password","cookies"}`` that are non-empty
+        """
+        out: dict[str, str] = {}
+        for key in ("username", "password", "cookies"):
+            val = self._site_cfg.get(key, "")
+            if isinstance(val, str):
+                s = val.strip()
+                if s:
+                    out[key] = s
+        return out
+
     def get_book_ids(self) -> list[BookConfig]:
         """
-        Extract the list of target books from the site configuration.
+        Extract and normalize the list of target books for the current site.
 
-        The site config may specify book_ids as:
-          - a single string or integer
-          - a dict with book_id and optional start_id, end_id, ignore_ids
-          - a list of the above types
+        Accepted shapes for ``site.book_ids``:
+          * a single ``str`` or ``int`` (book id)
+          * a dict  with fields: book_id and optional start_id, end_id, ignore_ids
+          * a ``list`` containing any mix of the above
 
-        :return: A list of BookConfig dicts.
-        :raises ValueError: if the raw book_ids is neither a str/int, dict, nor list.
+        :return: Normalized list of :class:`BookConfig`-compatible dictionaries.
+        :raises ValueError: If ``book_ids`` is neither a scalar ``str|int``, ``dict``,
+                            nor ``list``.
         """
-        site_cfg = self._get_site_cfg()
-        raw = site_cfg.get("book_ids", [])
+        raw = self._site_cfg.get("book_ids", [])
 
-        if isinstance(raw, str | int):
+        if isinstance(raw, (str | int)):
             return [{"book_id": str(raw)}]
 
         if isinstance(raw, dict):
@@ -207,154 +183,175 @@ class ConfigAdapter:
         result: list[BookConfig] = []
         for item in raw:
             try:
-                if isinstance(item, str | int):
+                if isinstance(item, (str | int)):
                     result.append({"book_id": str(item)})
                 elif isinstance(item, dict):
                     result.append(self._dict_to_book_cfg(item))
             except ValueError:
                 continue
-
         return result
 
-    def get_log_level(self) -> LogLevel:
+    def get_log_level(self) -> str:
         """
-        Retrieve the logging level from [general.debug].
+        Retrieve the logging level from ``general.debug``.
 
-        Reads from config["general"]["debug"]["log_level"], defaulting to "INFO"
-        if not set or invalid.
-
-        :return: The configured LogLevel literal ("DEBUG", "INFO", "WARNING", "ERROR").
+        :return: One of ``"DEBUG"``, ``"INFO"``, ``"WARNING"``, ``"ERROR"``
         """
-        debug_cfg = self._config.get("general", {}).get("debug", {})
-        raw = debug_cfg.get("log_level") or "INFO"
-        if raw in self._ALLOWED_LOG_LEVELS:
-            return cast(LogLevel, raw)
-        return "INFO"
+        debug_cfg = self._gen_cfg.get("debug", {})
+        return debug_cfg.get("log_level") or "INFO"
 
     @property
     def site(self) -> str:
-        """
-        Get the current site name.
-        """
         return self._site
 
     @site.setter
     def site(self, value: str) -> None:
+        self._site = value
+
+    @property
+    def _gen_cfg(self) -> dict[str, Any]:
         """
-        Set a new site name for configuration lookups.
+        A read-only view of the global ``general`` settings.
 
-        :param value: The new site key in config["sites"] to use.
+        :return: ``config["general"]`` if present, else ``{}``.
         """
-        self._site = value
+        return self._config.get("general") or {}
 
-    def _get_site_cfg(self, site: str | None = None) -> dict[str, Any]:
+    @property
+    def _site_cfg(self) -> dict[str, Any]:
         """
-        Retrieve the configuration for a specific site.
+        Retrieve the configuration block for the current site.
 
         Lookup order:
-          1. If there is a site-specific entry under config["sites"], return that.
-          2. Otherwise, if a "common" entry exists under config["sites"], return that.
-          3. If neither is present, return an empty dict.
+          1. If a site-specific entry exists under ``config["sites"]``, return it.
+          2. Otherwise, if ``config["sites"]["common"]`` exists, return it.
+          3. Else return an empty dict.
+
+        :return: Site-specific mapping, common mapping, or ``{}``.
+        """
+        sites_cfg = self._config.get("sites") or {}
+        if self._site in sites_cfg and isinstance(sites_cfg[self._site], dict):
+            return sites_cfg[self._site] or {}
+        return sites_cfg.get("common") or {}
+
+    @staticmethod
+    def _has_key(d: Mapping[str, Any] | None, key: str) -> bool:
+        """
+        Check whether a mapping contains a key.
 
-        :param site: Optional override of the site name; defaults to self._site.
-        :return: The site-specific or common configuration dict.
+        :param d: Mapping to inspect.
+        :param key: Key to look up.
+        :return: ``True`` if ``d`` is a Mapping and contains key; otherwise ``False``.
         """
-        site = site or self._site
-        sites_cfg = self._config.get("sites", {}) or {}
+        return isinstance(d, Mapping) and (key in d)
 
-        if site in sites_cfg:
-            return sites_cfg[site] or {}
+    def _pick(self, key: str, default: T, *sources: Mapping[str, Any]) -> T:
+        """
+        Resolve ``key`` from the provided ``sources`` in order of precedence.
 
-        return sites_cfg.get("common", {}) or {}
+        :param key: Configuration key to resolve.
+        :param default: Fallback value if ``key`` is absent in all sources.
+        :param sources: One or more mappings to check, in order of precedence.
+        :return: The first present value for ``key``, otherwise ``default``.
+        """
+        for src in sources:
+            if self._has_key(src, key):
+                return src[key]  # type: ignore[no-any-return]
+        return default
 
     @staticmethod
     def _dict_to_book_cfg(data: dict[str, Any]) -> BookConfig:
         """
-        Convert a dictionary to a BookConfig with normalized types.
+        Convert a raw dict into a :class:`novel_downloader.models.BookConfig`
+        with normalized types (all IDs coerced to strings).
 
         :param data: A dict that must contain at least "book_id".
-        :return: A BookConfig dict with all values cast to strings or lists of strings.
-        :raises ValueError: if the "book_id" field is missing.
+        :return: Normalized :class:`BookConfig` mapping.
+        :raises ValueError: If ``"book_id"`` is missing.
         """
         if "book_id" not in data:
             raise ValueError("Missing required field 'book_id'")
 
-        result: BookConfig = {"book_id": str(data["book_id"])}
+        out: BookConfig = {"book_id": str(data["book_id"])}
 
         if "start_id" in data:
-            result["start_id"] = str(data["start_id"])
-
+            out["start_id"] = str(data["start_id"])
         if "end_id" in data:
-            result["end_id"] = str(data["end_id"])
-
+            out["end_id"] = str(data["end_id"])
         if "ignore_ids" in data:
-            result["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
-
-        return result
+            with contextlib.suppress(Exception):
+                out["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
+        return out
 
     @classmethod
     def _dict_to_cleaner_cfg(cls, cfg: dict[str, Any]) -> TextCleanerConfig:
         """
-        Convert a nested dict of title/content rules into a TextCleanerConfig.
+        Convert a nested ``cleaner`` block into a
+        :class:`novel_downloader.models.TextCleanerConfig`.
 
         :param cfg: configuration dictionary
-        :return: fully constructed TextCleanerConfig
+        :return: Aggregated title/content rules with external file contents merged
         """
-        # Title rules
-        title_section = cfg.get("title", {})
-        title_remove = title_section.get("remove_patterns", [])
-        title_repl = title_section.get("replace", {})
-
-        title_ext = title_section.get("external", {})
-        title_ext_en = title_ext.get("enabled", False)
-        title_ext_rm_p = title_ext.get("remove_patterns", "")
-        title_ext_rp_p = title_ext.get("replace", "")
-        if title_ext_en:
-            title_remove_ext = cls._load_str_list(title_ext_rm_p)
-            title_remove += title_remove_ext
-
-            title_repl_ext = cls._load_str_dict(title_ext_rp_p)
-            title_repl = {**title_repl, **title_repl_ext}
-
-        # Content rules
-        content_section = cfg.get("content", {})
-        content_remove = content_section.get("remove_patterns", [])
-        content_repl = content_section.get("replace", {})
-
-        content_ext = content_section.get("external", {})
-        content_ext_en = content_ext.get("enabled", False)
-        content_ext_rm_p = content_ext.get("remove_patterns", "")
-        content_ext_rp_p = content_ext.get("replace", "")
-
-        if content_ext_en:
-            content_remove_ext = cls._load_str_list(content_ext_rm_p)
-            content_remove += content_remove_ext
-
-            content_repl_ext = cls._load_str_dict(content_ext_rp_p)
-            content_repl = {**content_repl, **content_repl_ext}
-
+        t_remove, t_replace = cls._merge_rules(cfg.get("title", {}) or {})
+        c_remove, c_replace = cls._merge_rules(cfg.get("content", {}) or {})
         return TextCleanerConfig(
             remove_invisible=cfg.get("remove_invisible", True),
-            title_remove_patterns=title_remove,
-            title_replacements=title_repl,
-            content_remove_patterns=content_remove,
-            content_replacements=content_repl,
+            title_remove_patterns=t_remove,
+            title_replacements=t_replace,
+            content_remove_patterns=c_remove,
+            content_replacements=c_replace,
         )
 
+    @classmethod
+    def _merge_rules(cls, section: dict[str, Any]) -> tuple[list[str], dict[str, str]]:
+        """
+        Merge inline patterns/replacements with any enabled external files.
+
+        :param section: Mapping describing either the ``title`` or ``content`` rules.
+        :return: Tuple ``(remove_patterns, replace)`` after merging.
+        """
+        remove = list(section.get("remove_patterns") or [])
+        replace = dict(section.get("replace") or {})
+        ext = section.get("external") or {}
+        if ext.get("enabled", False):
+            rm_path = ext.get("remove_patterns") or ""
+            rp_path = ext.get("replace") or ""
+            remove += cls._load_str_list(rm_path)
+            replace.update(cls._load_str_dict(rp_path))
+        return remove, replace
+
     @staticmethod
     def _load_str_list(path: str) -> list[str]:
+        """
+        Load a JSON file containing a list of strings.
+
+        :param path: File path to a JSON array (e.g., ``["a", "b"]``).
+        :return: Parsed list on success; empty list if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return []
         try:
             with open(path, encoding="utf-8") as f:
-                parsed = json.load(f)
-            return cast(list[str], parsed)
+                data = json.load(f)
+                return list(data) if isinstance(data, list) else []
         except Exception:
             return []
 
     @staticmethod
     def _load_str_dict(path: str) -> dict[str, str]:
+        """
+        Load a JSON file containing a dict of string-to-string mappings.
+
+        :param path: File path to a JSON object (e.g., ``{"old":"new"}``).
+        :return: Parsed dict on success; empty dict if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return {}
         try:
             with open(path, encoding="utf-8") as f:
-                parsed = json.load(f)
-            return cast(dict[str, str], parsed)
+                data = json.load(f)
+                return dict(data) if isinstance(data, dict) else {}
         except Exception:
             return {}