novel-downloader 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

novel_downloader/config/adapter.py

@@ -4,17 +4,20 @@ novel_downloader.config.adapter
 -------------------------------
 
 Defines ConfigAdapter, which maps a raw configuration dictionary and
-site name into structured dataclass-based config models.
+site into structured dataclass-based config models.
 """
 
+import contextlib
 import json
-from typing import Any, TypeVar, cast
+from collections.abc import Mapping
+from typing import Any, TypeVar
 
 from novel_downloader.models import (
     BookConfig,
     DownloaderConfig,
     ExporterConfig,
     FetcherConfig,
+    FontOCRConfig,
     ParserConfig,
     TextCleanerConfig,
 )
@@ -26,96 +29,109 @@ class ConfigAdapter:
     """
     Adapter to map a raw configuration dictionary and site name
     into structured dataclass configuration models.
+
+    Resolution order for each field:
+      1. ``config["sites"][<site>]`` (if present)
+      2. ``config["general"]`` (if present)
+      3. Hard-coded default passed by the caller
     """
 
-    def __init__(self, config: dict[str, Any], site: str):
+    def __init__(self, config: Mapping[str, Any], site: str):
         """
-        Initialize the adapter.
+        Initialize the adapter with a configuration mapping and a site key.
 
-        :param config: The fully loaded configuration dictionary.
-        :param site: The current site name (e.g. "qidian").
+        :param config: Fully loaded configuration mapping.
+        :param site: Current site key (e.g., ``"qidian"``).
         """
-        self._config = config
-        self._site = site
-        self._site_cfg: dict[str, Any] = self._get_site_cfg()
-        self._gen_cfg: dict[str, Any] = config.get("general") or {}
+        self._config: dict[str, Any] = dict(config)
+        self._site: str = site
 
     def get_fetcher_config(self) -> FetcherConfig:
         """
-        Build a FetcherConfig from the raw configuration.
+        Build a :class:`novel_downloader.models.FetcherConfig` by resolving fields
+        from site-specific and general settings.
 
-        :return: A FetcherConfig instance with all fields populated.
+        :return: Fully populated configuration for the network fetcher.
         """
+        s, g = self._site_cfg, self._gen_cfg
         return FetcherConfig(
-            request_interval=self._get_gen_cfg("request_interval", 2.0),
-            retry_times=self._get_gen_cfg("retry_times", 3),
-            backoff_factor=self._get_gen_cfg("backoff_factor", 2.0),
-            timeout=self._get_gen_cfg("timeout", 30.0),
-            max_connections=self._get_gen_cfg("max_connections", 10),
-            max_rps=self._get_gen_cfg("max_rps", 1000.0),
-            user_agent=self._get_gen_cfg("user_agent", None),
-            headers=self._get_gen_cfg("headers", None),
-            verify_ssl=self._get_gen_cfg("verify_ssl", True),
-            locale_style=self._get_gen_cfg("locale_style", "simplified"),
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            timeout=self._pick("timeout", 30.0, s, g),
+            max_connections=self._pick("max_connections", 10, s, g),
+            max_rps=self._pick("max_rps", 1000.0, s, g),
+            user_agent=self._pick("user_agent", None, s, g),
+            headers=self._pick("headers", None, s, g),
+            verify_ssl=self._pick("verify_ssl", True, s, g),
+            locale_style=self._pick("locale_style", "simplified", s, g),
         )
 
     def get_downloader_config(self) -> DownloaderConfig:
         """
-        Build a DownloaderConfig using both general and site-specific settings.
+        Build a :class:`novel_downloader.models.DownloaderConfig` using both
+        general and site-specific settings.
 
-        :return: A DownloaderConfig instance with all fields populated.
+        :return: Fully populated configuration for the chapter/page downloader.
         """
-        gen = self._config.get("general", {})
-        debug = gen.get("debug", {})
+        s, g = self._site_cfg, self._gen_cfg
+        debug = g.get("debug") or {}
         return DownloaderConfig(
-            request_interval=self._get_gen_cfg("request_interval", 2.0),
-            retry_times=self._get_gen_cfg("retry_times", 3),
-            backoff_factor=self._get_gen_cfg("backoff_factor", 2.0),
-            workers=self._get_gen_cfg("workers", 2),
-            skip_existing=self._get_gen_cfg("skip_existing", True),
-            login_required=self._site_cfg.get("login_required", False),
-            save_html=debug.get("save_html", False),
-            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            storage_batch_size=gen.get("storage_batch_size", 1),
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            workers=self._pick("workers", 2, s, g),
+            skip_existing=self._pick("skip_existing", True, s, g),
+            login_required=bool(s.get("login_required", False)),
+            save_html=bool(debug.get("save_html", False)),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            storage_batch_size=g.get("storage_batch_size", 1),
         )
 
     def get_parser_config(self) -> ParserConfig:
         """
-        Build a ParserConfig from general, OCR, and site-specific settings.
+        Build a :class:`novel_downloader.models.ParserConfig` from general,
+        OCR-related, and site-specific settings.
 
-        :return: A ParserConfig instance with all fields populated.
+        :return: Fully populated configuration for the parser stage.
         """
-        gen = self._config.get("general", {})
-        font_ocr = gen.get("font_ocr", {})
+        g = self._gen_cfg
+        s = self._site_cfg
+        g_font = g.get("font_ocr") or {}
+        s_font = s.get("font_ocr") or {}
+        font_ocr: dict[str, Any] = {**g_font, **s_font}
         return ParserConfig(
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            use_truncation=self._site_cfg.get("use_truncation", True),
-            decode_font=font_ocr.get("decode_font", False),
-            save_font_debug=font_ocr.get("save_font_debug", False),
-            batch_size=font_ocr.get("batch_size", 32),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            use_truncation=bool(s.get("use_truncation", True)),
+            decode_font=bool(font_ocr.get("decode_font", False)),
+            save_font_debug=bool(font_ocr.get("save_font_debug", False)),
+            batch_size=int(font_ocr.get("batch_size", 32)),
+            fontocr_cfg=self._dict_to_fontocr_cfg(font_ocr),
         )
 
     def get_exporter_config(self) -> ExporterConfig:
         """
-        Build an ExporterConfig from output and general settings.
+        Build an :class:`novel_downloader.models.ExporterConfig` from the
+        ``output`` and ``cleaner`` sections plus general settings.
 
-        :return: An ExporterConfig instance with all fields populated.
+        :return: Fully populated configuration for text/ebook export.
         """
-        gen = self._config.get("general", {})
-        out = self._config.get("output", {})
-        cln = self._config.get("cleaner", {})
-        fmt = out.get("formats", {})
-        naming = out.get("naming", {})
-        epub_opts = out.get("epub", {})
+        g = self._gen_cfg
+        out = self._config.get("output") or {}
+        cln = self._config.get("cleaner") or {}
+        fmt = out.get("formats") or {}
+        naming = out.get("naming") or {}
+        epub_opts = out.get("epub") or {}
+
         cleaner_cfg = self._dict_to_cleaner_cfg(cln)
         return ExporterConfig(
-            cache_dir=gen.get("cache_dir", "./novel_cache"),
-            raw_data_dir=gen.get("raw_data_dir", "./raw_data"),
-            output_dir=gen.get("output_dir", "./downloads"),
-            clean_text=cln.get("clean_text", True),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            output_dir=g.get("output_dir", "./downloads"),
+            clean_text=cln.get("clean_text", False),
             make_txt=fmt.get("make_txt", True),
-            make_epub=fmt.get("make_epub", False),
+            make_epub=fmt.get("make_epub", True),
             make_md=fmt.get("make_md", False),
             make_pdf=fmt.get("make_pdf", False),
             append_timestamp=naming.get("append_timestamp", True),
@@ -128,35 +144,36 @@ class ConfigAdapter:
 
     def get_login_config(self) -> dict[str, str]:
         """
-        Return the subset of login fields present in current site config:
-            * `username`
-            * `password`
-            * `cookies`
+        Extract login-related fields from the current site configuration.
+        Only non-empty string values are returned; values are stripped.
+
+        :return: A subset of ``{"username","password","cookies"}`` that are non-empty
         """
         out: dict[str, str] = {}
         for key in ("username", "password", "cookies"):
             val = self._site_cfg.get(key, "")
-            val = val.strip()
-            if val:
-                out[key] = val
+            if isinstance(val, str):
+                s = val.strip()
+                if s:
+                    out[key] = s
         return out
 
     def get_book_ids(self) -> list[BookConfig]:
         """
-        Extract the list of target books from the site configuration.
+        Extract and normalize the list of target books for the current site.
 
-        The site config may specify book_ids as:
-          * a single string or integer
-          * a dict with book_id and optional start_id, end_id, ignore_ids
-          * a list of the above types
+        Accepted shapes for ``site.book_ids``:
+          * a single ``str`` or ``int`` (book id)
+          * a dict  with fields: book_id and optional start_id, end_id, ignore_ids
+          * a ``list`` containing any mix of the above
 
-        :return: A list of BookConfig dicts.
-        :raises ValueError: if the raw book_ids is neither a str/int, dict, nor list.
+        :return: Normalized list of :class:`BookConfig`-compatible dictionaries.
+        :raises ValueError: If ``book_ids`` is neither a scalar ``str|int``, ``dict``,
+                            nor ``list``.
         """
-        site_cfg = self._get_site_cfg()
-        raw = site_cfg.get("book_ids", [])
+        raw = self._site_cfg.get("book_ids", [])
 
-        if isinstance(raw, str | int):
+        if isinstance(raw, (str | int)):
             return [{"book_id": str(raw)}]
 
         if isinstance(raw, dict):
@@ -170,151 +187,197 @@ class ConfigAdapter:
         result: list[BookConfig] = []
         for item in raw:
             try:
-                if isinstance(item, str | int):
+                if isinstance(item, (str | int)):
                     result.append({"book_id": str(item)})
                 elif isinstance(item, dict):
                     result.append(self._dict_to_book_cfg(item))
             except ValueError:
                 continue
-
         return result
 
     def get_log_level(self) -> str:
         """
-        Retrieve the logging level from [general.debug].
+        Retrieve the logging level from ``general.debug``.
 
-        :return: The configured log level ("DEBUG", "INFO", "WARNING", "ERROR").
+        :return: One of ``"DEBUG"``, ``"INFO"``, ``"WARNING"``, ``"ERROR"``
         """
-        debug_cfg = self._config.get("general", {}).get("debug", {})
+        debug_cfg = self._gen_cfg.get("debug", {})
         return debug_cfg.get("log_level") or "INFO"
 
     @property
     def site(self) -> str:
-        """
-        Get the current site name.
-        """
         return self._site
 
     @site.setter
     def site(self, value: str) -> None:
-        """
-        Set a new site name for configuration lookups.
+        self._site = value
 
-        :param value: The new site key in config["sites"] to use.
+    @property
+    def _gen_cfg(self) -> dict[str, Any]:
         """
-        self._site = value
-        self._site_cfg = self._get_site_cfg()
+        A read-only view of the global ``general`` settings.
 
-    def _get_gen_cfg(self, key: str, default: T) -> T:
-        return self._site_cfg.get(key) or self._gen_cfg.get(key) or default
+        :return: ``config["general"]`` if present, else ``{}``.
+        """
+        return self._config.get("general") or {}
 
-    def _get_site_cfg(self) -> dict[str, Any]:
+    @property
+    def _site_cfg(self) -> dict[str, Any]:
         """
-        Retrieve the configuration for a specific site.
+        Retrieve the configuration block for the current site.
 
         Lookup order:
-          1. If there is a site-specific entry under config["sites"], return that.
-          2. Otherwise, if a "common" entry exists under config["sites"], return that.
-          3. If neither is present, return an empty dict.
+          1. If a site-specific entry exists under ``config["sites"]``, return it.
+          2. Otherwise, if ``config["sites"]["common"]`` exists, return it.
+          3. Else return an empty dict.
 
-        :param site: Optional override of the site name; defaults to self._site.
-        :return: The site-specific or common configuration dict.
+        :return: Site-specific mapping, common mapping, or ``{}``.
         """
         sites_cfg = self._config.get("sites") or {}
-
-        if self._site in sites_cfg:
+        if self._site in sites_cfg and isinstance(sites_cfg[self._site], dict):
             return sites_cfg[self._site] or {}
-
         return sites_cfg.get("common") or {}
 
+    @staticmethod
+    def _has_key(d: Mapping[str, Any] | None, key: str) -> bool:
+        """
+        Check whether a mapping contains a key.
+
+        :param d: Mapping to inspect.
+        :param key: Key to look up.
+        :return: ``True`` if ``d`` is a Mapping and contains key; otherwise ``False``.
+        """
+        return isinstance(d, Mapping) and (key in d)
+
+    def _pick(self, key: str, default: T, *sources: Mapping[str, Any]) -> T:
+        """
+        Resolve ``key`` from the provided ``sources`` in order of precedence.
+
+        :param key: Configuration key to resolve.
+        :param default: Fallback value if ``key`` is absent in all sources.
+        :param sources: One or more mappings to check, in order of precedence.
+        :return: The first present value for ``key``, otherwise ``default``.
+        """
+        for src in sources:
+            if self._has_key(src, key):
+                return src[key]  # type: ignore[no-any-return]
+        return default
+
     @staticmethod
     def _dict_to_book_cfg(data: dict[str, Any]) -> BookConfig:
         """
-        Convert a dictionary to a BookConfig with normalized types.
+        Convert a raw dict into a :class:`novel_downloader.models.BookConfig`
+        with normalized types (all IDs coerced to strings).
 
         :param data: A dict that must contain at least "book_id".
-        :return: A BookConfig dict with all values cast to strings or lists of strings.
-        :raises ValueError: if the "book_id" field is missing.
+        :return: Normalized :class:`BookConfig` mapping.
+        :raises ValueError: If ``"book_id"`` is missing.
         """
         if "book_id" not in data:
             raise ValueError("Missing required field 'book_id'")
 
-        result: BookConfig = {"book_id": str(data["book_id"])}
+        out: BookConfig = {"book_id": str(data["book_id"])}
 
         if "start_id" in data:
-            result["start_id"] = str(data["start_id"])
-
+            out["start_id"] = str(data["start_id"])
         if "end_id" in data:
-            result["end_id"] = str(data["end_id"])
-
+            out["end_id"] = str(data["end_id"])
         if "ignore_ids" in data:
-            result["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
+            with contextlib.suppress(Exception):
+                out["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
+        return out
 
-        return result
+    @staticmethod
+    def _dict_to_fontocr_cfg(data: dict[str, Any]) -> FontOCRConfig:
+        """
+        Convert a raw ``font_ocr`` dict into a :class:`FontOCRConfig`.
+        """
+        if not isinstance(data, dict):
+            return FontOCRConfig()
+
+        ishape = data.get("input_shape")
+        if isinstance(ishape, list):
+            ishape = tuple(ishape)  # [C, H, W] -> (C, H, W)
+
+        return FontOCRConfig(
+            model_name=data.get("model_name"),
+            model_dir=data.get("model_dir"),
+            input_shape=ishape,
+            device=data.get("device"),
+            precision=data.get("precision", "fp32"),
+            cpu_threads=data.get("cpu_threads", 10),
+            enable_hpi=data.get("enable_hpi", False),
+        )
 
     @classmethod
     def _dict_to_cleaner_cfg(cls, cfg: dict[str, Any]) -> TextCleanerConfig:
         """
-        Convert a nested dict of title/content rules into a TextCleanerConfig.
+        Convert a nested ``cleaner`` block into a
+        :class:`novel_downloader.models.TextCleanerConfig`.
 
         :param cfg: configuration dictionary
-        :return: fully constructed TextCleanerConfig
+        :return: Aggregated title/content rules with external file contents merged
         """
-        # Title rules
-        title_section = cfg.get("title", {})
-        title_remove = title_section.get("remove_patterns", [])
-        title_repl = title_section.get("replace", {})
-
-        title_ext = title_section.get("external", {})
-        if title_ext.get("enabled", False):
-            title_ext_rm_p = title_ext.get("remove_patterns", "")
-            title_ext_rp_p = title_ext.get("replace", "")
-
-            title_remove_ext = cls._load_str_list(title_ext_rm_p)
-            title_remove += title_remove_ext
-
-            title_repl_ext = cls._load_str_dict(title_ext_rp_p)
-            title_repl = {**title_repl, **title_repl_ext}
-
-        # Content rules
-        content_section = cfg.get("content", {})
-        content_remove = content_section.get("remove_patterns", [])
-        content_repl = content_section.get("replace", {})
-
-        content_ext = content_section.get("external", {})
-
-        if content_ext.get("enabled", False):
-            content_ext_rm_p = content_ext.get("remove_patterns", "")
-            content_ext_rp_p = content_ext.get("replace", "")
-
-            content_remove_ext = cls._load_str_list(content_ext_rm_p)
-            content_remove += content_remove_ext
-
-            content_repl_ext = cls._load_str_dict(content_ext_rp_p)
-            content_repl = {**content_repl, **content_repl_ext}
-
+        t_remove, t_replace = cls._merge_rules(cfg.get("title", {}) or {})
+        c_remove, c_replace = cls._merge_rules(cfg.get("content", {}) or {})
         return TextCleanerConfig(
             remove_invisible=cfg.get("remove_invisible", True),
-            title_remove_patterns=title_remove,
-            title_replacements=title_repl,
-            content_remove_patterns=content_remove,
-            content_replacements=content_repl,
+            title_remove_patterns=t_remove,
+            title_replacements=t_replace,
+            content_remove_patterns=c_remove,
+            content_replacements=c_replace,
         )
 
+    @classmethod
+    def _merge_rules(cls, section: dict[str, Any]) -> tuple[list[str], dict[str, str]]:
+        """
+        Merge inline patterns/replacements with any enabled external files.
+
+        :param section: Mapping describing either the ``title`` or ``content`` rules.
+        :return: Tuple ``(remove_patterns, replace)`` after merging.
+        """
+        remove = list(section.get("remove_patterns") or [])
+        replace = dict(section.get("replace") or {})
+        ext = section.get("external") or {}
+        if ext.get("enabled", False):
+            rm_path = ext.get("remove_patterns") or ""
+            rp_path = ext.get("replace") or ""
+            remove += cls._load_str_list(rm_path)
+            replace.update(cls._load_str_dict(rp_path))
+        return remove, replace
+
     @staticmethod
     def _load_str_list(path: str) -> list[str]:
+        """
+        Load a JSON file containing a list of strings.
+
+        :param path: File path to a JSON array (e.g., ``["a", "b"]``).
+        :return: Parsed list on success; empty list if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return []
         try:
             with open(path, encoding="utf-8") as f:
-                parsed = json.load(f)
-            return cast(list[str], parsed)
+                data = json.load(f)
+                return list(data) if isinstance(data, list) else []
         except Exception:
             return []
 
     @staticmethod
     def _load_str_dict(path: str) -> dict[str, str]:
+        """
+        Load a JSON file containing a dict of string-to-string mappings.
+
+        :param path: File path to a JSON object (e.g., ``{"old":"new"}``).
+        :return: Parsed dict on success; empty dict if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return {}
         try:
             with open(path, encoding="utf-8") as f:
-                parsed = json.load(f)
-            return cast(dict[str, str], parsed)
+                data = json.load(f)
+                return dict(data) if isinstance(data, dict) else {}
         except Exception:
             return {}

novel_downloader/core/__init__.py

@@ -7,12 +7,11 @@ This package serves as the core layer of the novel_downloader system.
 
 It provides factory methods for constructing key components required for
 downloading and processing online novel content, including:
-
-- Downloader: Handles the full download lifecycle of a book or a batch of books.
-- Parser: Extracts structured data from HTML or SSR content.
-- Fetcher: Sends HTTP requests and manages sessions, including login if required.
-- Exporter: Responsible for exporting downloaded data into various output formats.
-- search: Provides unified search functionality across supported novel sites.
+  * Downloader: Handles the full download lifecycle of a book or a batch of books.
+  * Parser: Extracts structured data from HTML or SSR content.
+  * Fetcher: Sends HTTP requests and manages sessions, including login if required.
+  * Exporter: Responsible for exporting downloaded data into various output formats.
+  * search: Provides unified search functionality across supported novel sites.
 """
 
 __all__ = [

novel_downloader/core/archived/deqixs/fetcher.py

@@ -8,8 +8,6 @@ novel_downloader.core.archived.deqixs.fetcher
 from typing import Any
 
 from novel_downloader.core.fetchers.base import BaseSession
-from novel_downloader.models import FetcherConfig
-from novel_downloader.utils import async_jitter_sleep
 
 # from novel_downloader.core.fetchers.registry import register_fetcher
 
@@ -26,25 +24,11 @@ class DeqixsSession(BaseSession):
     BOOK_INFO_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/"
     CHAPTER_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/{chapter_id}.html"
 
-    def __init__(
-        self,
-        config: FetcherConfig,
-        cookies: dict[str, str] | None = None,
-        **kwargs: Any,
-    ) -> None:
-        super().__init__("deqixs", config, cookies, **kwargs)
-
     async def get_book_info(
         self,
         book_id: str,
         **kwargs: Any,
     ) -> list[str]:
-        """
-        Fetch the raw HTML of the book info page asynchronously.
-
-        :param book_id: The book identifier.
-        :return: The page content as a string.
-        """
         url = self.book_info_url(book_id=book_id)
         return [await self.fetch(url, **kwargs)]
 
@@ -54,13 +38,6 @@ class DeqixsSession(BaseSession):
         chapter_id: str,
         **kwargs: Any,
     ) -> list[str]:
-        """
-        Fetch the raw HTML of a single chapter asynchronously.
-
-        :param book_id: The book identifier.
-        :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
-        """
         html_pages: list[str] = []
         idx = 1
 
@@ -85,11 +62,7 @@ class DeqixsSession(BaseSession):
 
             html_pages.append(html)
             idx += 1
-            await async_jitter_sleep(
-                self.request_interval,
-                mul_spread=1.1,
-                max_sleep=self.request_interval + 2,
-            )
+            await self._sleep()
 
         return html_pages
 

novel_downloader/core/downloaders/__init__.py

@@ -11,9 +11,11 @@ __all__ = [
     "CommonDownloader",
     "QianbiDownloader",
     "QidianDownloader",
+    "QqbookDownloader",
 ]
 
 from .common import CommonDownloader
 from .qianbi import QianbiDownloader
 from .qidian import QidianDownloader
+from .qqbook import QqbookDownloader
 from .registry import get_downloader