novel-downloader 2.0.0__tar.gz → 2.0.2__tar.gz

{novel_downloader-2.0.0/src/novel_downloader.egg-info → novel_downloader-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: novel-downloader
-Version: 2.0.0
+Version: 2.0.2
 Summary: A command-line tool for downloading Chinese web novels from Qidian and similar platforms.
 Author-email: Saudade Z <saudadez217@gmail.com>
 License: MIT License
@@ -47,6 +47,7 @@ Requires-Dist: aiohttp
 Requires-Dist: lxml
 Requires-Dist: platformdirs
 Provides-Extra: font-recovery
+Requires-Dist: brotli; extra == "font-recovery"
 Requires-Dist: numpy; extra == "font-recovery"
 Requires-Dist: fonttools; extra == "font-recovery"
 Requires-Dist: pillow; extra == "font-recovery"
@@ -122,6 +123,7 @@ novel-cli download 123456
 
 * 支持站点见: [支持站点列表](https://github.com/saudadez21/novel-downloader/blob/main/docs/4-supported-sites.md)
 * 更多示例见: [CLI 使用示例](https://github.com/saudadez21/novel-downloader/blob/main/docs/5-cli-usage-examples.md)
+* 运行中可使用 `CTRL+C` 取消任务
 
 ### 3. 图形界面 (GUI / Web)
 
@@ -133,6 +135,8 @@ novel-web
 # novel-web --listen public
 ```
 
+* 运行中可使用 `CTRL+C` 停止服务
+
 ---
 
 ## 从源码安装 (开发版)

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/README.md

@@ -56,6 +56,7 @@ novel-cli download 123456
 
 * 支持站点见: [支持站点列表](https://github.com/saudadez21/novel-downloader/blob/main/docs/4-supported-sites.md)
 * 更多示例见: [CLI 使用示例](https://github.com/saudadez21/novel-downloader/blob/main/docs/5-cli-usage-examples.md)
+* 运行中可使用 `CTRL+C` 取消任务
 
 ### 3. 图形界面 (GUI / Web)
 
@@ -67,6 +68,8 @@ novel-web
 # novel-web --listen public
 ```
 
+* 运行中可使用 `CTRL+C` 停止服务
+
 ---
 
 ## 从源码安装 (开发版)

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/pyproject.toml

@@ -31,6 +31,7 @@ dependencies = [
 
 [project.optional-dependencies]
 font-recovery = [
+    "brotli",
     "numpy",
     "fonttools",
     "pillow",
@@ -110,7 +111,7 @@ testpaths = ["tests"]
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "2.0.0"
+version = "2.0.2"
 version_files = [
     "src/novel_downloader/__init__.py"
 ]

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/src/novel_downloader/__init__.py

@@ -6,7 +6,7 @@ novel_downloader
 Core package for the Novel Downloader project.
 """
 
-__version__ = "2.0.0"
+__version__ = "2.0.2"
 
 __author__ = "Saudade Z"
 __email__ = "saudadez217@gmail.com"

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/src/novel_downloader/cli/download.py

@@ -155,7 +155,7 @@ async def _download(
     exporter_cfg = adapter.get_exporter_config()
     login_cfg = adapter.get_login_config()
     log_level = adapter.get_log_level()
-    setup_logging(log_level=log_level)
+    setup_logging(console_level=log_level)
 
     parser = get_parser(site, parser_cfg)
     exporter = None
@@ -178,7 +178,14 @@ async def _download(
 
         for book in valid_books:
             ui.info(t("download_downloading", book_id=book["book_id"], site=site))
-            await downloader.download(book, progress_hook=_print_progress)
+
+            hook, close = ui.create_progress_hook(
+                prefix=t("download_progress_prefix"), unit="chapters"
+            )
+            try:
+                await downloader.download(book, progress_hook=hook)
+            finally:
+                close()
 
             if not no_export and exporter is not None:
                 await asyncio.to_thread(exporter.export, book["book_id"])
@@ -186,6 +193,9 @@ async def _download(
         if downloader_cfg.login_required and fetcher.is_logged_in:
             await fetcher.save_state()
 
+    if exporter is not None:
+        exporter.close()
+
 
 async def _prompt_login_fields(
     fields: list[LoginField],
@@ -214,8 +224,8 @@ async def _prompt_login_fields(
             ui.info(t("login_use_config"))
             continue
 
-        value: str | dict[str, str]
-        while True:
+        value: str | dict[str, str] = ""
+        for _ in range(5):
             if field.type == "password":
                 value = ui.prompt_password(t("login_enter_password"))
             elif field.type == "cookie":
@@ -235,10 +245,3 @@ async def _prompt_login_fields(
         result[field.name] = value
 
     return result
-
-
-async def _print_progress(done: int, total: int) -> None:
-    """Progress hook passed into the downloader."""
-    ui.print_progress(
-        done, total, prefix=t("download_progress_prefix"), unit="chapters"
-    )

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/src/novel_downloader/cli/export.py

@@ -55,22 +55,22 @@ def handle_export(args: Namespace) -> None:
     adapter = ConfigAdapter(config=config_data, site=site)
     exporter_cfg = adapter.get_exporter_config()
     log_level = adapter.get_log_level()
-    exporter = get_exporter(site, exporter_cfg)
-    setup_logging(log_level=log_level)
-
-    for book_id in book_ids:
-        ui.info(t("export_processing", book_id=book_id, format=export_format))
-
-        if export_format in {"txt", "all"}:
-            try:
-                exporter.export_as_txt(book_id)
-                ui.success(t("export_success_txt", book_id=book_id))
-            except Exception as e:
-                ui.error(t("export_failed_txt", book_id=book_id, err=str(e)))
-
-        if export_format in {"epub", "all"}:
-            try:
-                exporter.export_as_epub(book_id)
-                ui.success(t("export_success_epub", book_id=book_id))
-            except Exception as e:
-                ui.error(t("export_failed_epub", book_id=book_id, err=str(e)))
+    setup_logging(console_level=log_level)
+
+    with get_exporter(site, exporter_cfg) as exporter:
+        for book_id in book_ids:
+            ui.info(t("export_processing", book_id=book_id, format=export_format))
+
+            if export_format in {"txt", "all"}:
+                try:
+                    exporter.export_as_txt(book_id)
+                    ui.success(t("export_success_txt", book_id=book_id))
+                except Exception as e:
+                    ui.error(t("export_failed_txt", book_id=book_id, err=str(e)))
+
+            if export_format in {"epub", "all"}:
+                try:
+                    exporter.export_as_epub(book_id)
+                    ui.success(t("export_success_epub", book_id=book_id))
+                except Exception as e:
+                    ui.error(t("export_failed_epub", book_id=book_id, err=str(e)))

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/src/novel_downloader/cli/ui.py

@@ -7,19 +7,20 @@ A small set of Rich-based helpers to keep CLI presentation and prompts
 consistent across subcommands.
 
 Public API:
-- info, success, warn, error
-- confirm
-- prompt, prompt_password
-- render_table
-- select_index
-- print_progress
+  * info, success, warn, error
+  * confirm
+  * prompt, prompt_password
+  * render_table
+  * select_index
+  * print_progress
 """
 
 from __future__ import annotations
 
-from collections.abc import Iterable, Sequence
+from collections.abc import Awaitable, Callable, Iterable, Sequence
 
 from rich.console import Console
+from rich.progress import Progress, TaskID
 from rich.prompt import Confirm, Prompt
 from rich.table import Table
 
@@ -71,7 +72,7 @@ def prompt(message: str, *, default: str | None = None) -> str:
     :return: The user's input.
     """
     try:
-        result: str = Prompt.ask(message, default=default or "")
+        result: str = Prompt.ask(message, default=default or "", show_default=False)
         return result
     except (KeyboardInterrupt, EOFError):
         warn("Cancelled.")
@@ -154,3 +155,29 @@ def print_progress(
     total = max(1, total)
     pct = done / total * 100.0
     _CONSOLE.print(f"[dim]{prefix}[/] {done}/{total} {unit} ({pct:.2f}%)")
+
+
+def create_progress_hook(
+    prefix: str = "Progress",
+    unit: str = "item",
+) -> tuple[Callable[[int, int], Awaitable[None]], Callable[[], None]]:
+    progress = Progress(console=_CONSOLE)
+    task_id: TaskID | None = None
+
+    async def hook(done: int, total: int) -> None:
+        nonlocal task_id
+        if task_id is None:
+            progress.start()
+            task_id = progress.add_task(f"[cyan]{prefix}[/]", total=max(1, total))
+
+        progress.update(
+            task_id,
+            completed=done,
+            total=max(1, total),
+            description=f"{prefix} ({done}/{total} {unit})",
+        )
+
+    def close() -> None:
+        progress.stop()
+
+    return hook, close

novel_downloader-2.0.2/src/novel_downloader/config/adapter.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.config.adapter
+-------------------------------
+
+Defines ConfigAdapter, which maps a raw configuration dictionary and
+site into structured dataclass-based config models.
+"""
+
+import contextlib
+import json
+from collections.abc import Mapping
+from typing import Any, TypeVar
+
+from novel_downloader.models import (
+    BookConfig,
+    DownloaderConfig,
+    ExporterConfig,
+    FetcherConfig,
+    FontOCRConfig,
+    ParserConfig,
+    TextCleanerConfig,
+)
+
+T = TypeVar("T")
+
+
+class ConfigAdapter:
+    """
+    Adapter to map a raw configuration dictionary and site name
+    into structured dataclass configuration models.
+
+    Resolution order for each field:
+      1. ``config["sites"][<site>]`` (if present)
+      2. ``config["general"]`` (if present)
+      3. Hard-coded default passed by the caller
+    """
+
+    def __init__(self, config: Mapping[str, Any], site: str):
+        """
+        Initialize the adapter with a configuration mapping and a site key.
+
+        :param config: Fully loaded configuration mapping.
+        :param site: Current site key (e.g., ``"qidian"``).
+        """
+        self._config: dict[str, Any] = dict(config)
+        self._site: str = site
+
+    def get_fetcher_config(self) -> FetcherConfig:
+        """
+        Build a :class:`novel_downloader.models.FetcherConfig` by resolving fields
+        from site-specific and general settings.
+
+        :return: Fully populated configuration for the network fetcher.
+        """
+        s, g = self._site_cfg, self._gen_cfg
+        return FetcherConfig(
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            timeout=self._pick("timeout", 30.0, s, g),
+            max_connections=self._pick("max_connections", 10, s, g),
+            max_rps=self._pick("max_rps", 1000.0, s, g),
+            user_agent=self._pick("user_agent", None, s, g),
+            headers=self._pick("headers", None, s, g),
+            verify_ssl=self._pick("verify_ssl", True, s, g),
+            locale_style=self._pick("locale_style", "simplified", s, g),
+        )
+
+    def get_downloader_config(self) -> DownloaderConfig:
+        """
+        Build a :class:`novel_downloader.models.DownloaderConfig` using both
+        general and site-specific settings.
+
+        :return: Fully populated configuration for the chapter/page downloader.
+        """
+        s, g = self._site_cfg, self._gen_cfg
+        debug = g.get("debug") or {}
+        return DownloaderConfig(
+            request_interval=self._pick("request_interval", 2.0, s, g),
+            retry_times=self._pick("retry_times", 3, s, g),
+            backoff_factor=self._pick("backoff_factor", 2.0, s, g),
+            workers=self._pick("workers", 2, s, g),
+            skip_existing=self._pick("skip_existing", True, s, g),
+            login_required=bool(s.get("login_required", False)),
+            save_html=bool(debug.get("save_html", False)),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            storage_batch_size=g.get("storage_batch_size", 1),
+        )
+
+    def get_parser_config(self) -> ParserConfig:
+        """
+        Build a :class:`novel_downloader.models.ParserConfig` from general,
+        OCR-related, and site-specific settings.
+
+        :return: Fully populated configuration for the parser stage.
+        """
+        g = self._gen_cfg
+        s = self._site_cfg
+        g_font = g.get("font_ocr") or {}
+        s_font = s.get("font_ocr") or {}
+        font_ocr: dict[str, Any] = {**g_font, **s_font}
+        return ParserConfig(
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            use_truncation=bool(s.get("use_truncation", True)),
+            decode_font=bool(font_ocr.get("decode_font", False)),
+            save_font_debug=bool(font_ocr.get("save_font_debug", False)),
+            batch_size=int(font_ocr.get("batch_size", 32)),
+            fontocr_cfg=self._dict_to_fontocr_cfg(font_ocr),
+        )
+
+    def get_exporter_config(self) -> ExporterConfig:
+        """
+        Build an :class:`novel_downloader.models.ExporterConfig` from the
+        ``output`` and ``cleaner`` sections plus general settings.
+
+        :return: Fully populated configuration for text/ebook export.
+        """
+        g = self._gen_cfg
+        out = self._config.get("output") or {}
+        cln = self._config.get("cleaner") or {}
+        fmt = out.get("formats") or {}
+        naming = out.get("naming") or {}
+        epub_opts = out.get("epub") or {}
+
+        cleaner_cfg = self._dict_to_cleaner_cfg(cln)
+        return ExporterConfig(
+            cache_dir=g.get("cache_dir", "./novel_cache"),
+            raw_data_dir=g.get("raw_data_dir", "./raw_data"),
+            output_dir=g.get("output_dir", "./downloads"),
+            clean_text=cln.get("clean_text", False),
+            make_txt=fmt.get("make_txt", True),
+            make_epub=fmt.get("make_epub", True),
+            make_md=fmt.get("make_md", False),
+            make_pdf=fmt.get("make_pdf", False),
+            append_timestamp=naming.get("append_timestamp", True),
+            filename_template=naming.get("filename_template", "{title}_{author}"),
+            include_cover=epub_opts.get("include_cover", True),
+            include_picture=epub_opts.get("include_picture", True),
+            split_mode=self._site_cfg.get("split_mode", "book"),
+            cleaner_cfg=cleaner_cfg,
+        )
+
+    def get_login_config(self) -> dict[str, str]:
+        """
+        Extract login-related fields from the current site configuration.
+        Only non-empty string values are returned; values are stripped.
+
+        :return: A subset of ``{"username","password","cookies"}`` that are non-empty
+        """
+        out: dict[str, str] = {}
+        for key in ("username", "password", "cookies"):
+            val = self._site_cfg.get(key, "")
+            if isinstance(val, str):
+                s = val.strip()
+                if s:
+                    out[key] = s
+        return out
+
+    def get_book_ids(self) -> list[BookConfig]:
+        """
+        Extract and normalize the list of target books for the current site.
+
+        Accepted shapes for ``site.book_ids``:
+          * a single ``str`` or ``int`` (book id)
+          * a dict  with fields: book_id and optional start_id, end_id, ignore_ids
+          * a ``list`` containing any mix of the above
+
+        :return: Normalized list of :class:`BookConfig`-compatible dictionaries.
+        :raises ValueError: If ``book_ids`` is neither a scalar ``str|int``, ``dict``,
+                            nor ``list``.
+        """
+        raw = self._site_cfg.get("book_ids", [])
+
+        if isinstance(raw, (str | int)):
+            return [{"book_id": str(raw)}]
+
+        if isinstance(raw, dict):
+            return [self._dict_to_book_cfg(raw)]
+
+        if not isinstance(raw, list):
+            raise ValueError(
+                f"book_ids must be a list or string, got {type(raw).__name__}"
+            )
+
+        result: list[BookConfig] = []
+        for item in raw:
+            try:
+                if isinstance(item, (str | int)):
+                    result.append({"book_id": str(item)})
+                elif isinstance(item, dict):
+                    result.append(self._dict_to_book_cfg(item))
+            except ValueError:
+                continue
+        return result
+
+    def get_log_level(self) -> str:
+        """
+        Retrieve the logging level from ``general.debug``.
+
+        :return: One of ``"DEBUG"``, ``"INFO"``, ``"WARNING"``, ``"ERROR"``
+        """
+        debug_cfg = self._gen_cfg.get("debug", {})
+        return debug_cfg.get("log_level") or "INFO"
+
+    @property
+    def site(self) -> str:
+        return self._site
+
+    @site.setter
+    def site(self, value: str) -> None:
+        self._site = value
+
+    @property
+    def _gen_cfg(self) -> dict[str, Any]:
+        """
+        A read-only view of the global ``general`` settings.
+
+        :return: ``config["general"]`` if present, else ``{}``.
+        """
+        return self._config.get("general") or {}
+
+    @property
+    def _site_cfg(self) -> dict[str, Any]:
+        """
+        Retrieve the configuration block for the current site.
+
+        Lookup order:
+          1. If a site-specific entry exists under ``config["sites"]``, return it.
+          2. Otherwise, if ``config["sites"]["common"]`` exists, return it.
+          3. Else return an empty dict.
+
+        :return: Site-specific mapping, common mapping, or ``{}``.
+        """
+        sites_cfg = self._config.get("sites") or {}
+        if self._site in sites_cfg and isinstance(sites_cfg[self._site], dict):
+            return sites_cfg[self._site] or {}
+        return sites_cfg.get("common") or {}
+
+    @staticmethod
+    def _has_key(d: Mapping[str, Any] | None, key: str) -> bool:
+        """
+        Check whether a mapping contains a key.
+
+        :param d: Mapping to inspect.
+        :param key: Key to look up.
+        :return: ``True`` if ``d`` is a Mapping and contains key; otherwise ``False``.
+        """
+        return isinstance(d, Mapping) and (key in d)
+
+    def _pick(self, key: str, default: T, *sources: Mapping[str, Any]) -> T:
+        """
+        Resolve ``key`` from the provided ``sources`` in order of precedence.
+
+        :param key: Configuration key to resolve.
+        :param default: Fallback value if ``key`` is absent in all sources.
+        :param sources: One or more mappings to check, in order of precedence.
+        :return: The first present value for ``key``, otherwise ``default``.
+        """
+        for src in sources:
+            if self._has_key(src, key):
+                return src[key]  # type: ignore[no-any-return]
+        return default
+
+    @staticmethod
+    def _dict_to_book_cfg(data: dict[str, Any]) -> BookConfig:
+        """
+        Convert a raw dict into a :class:`novel_downloader.models.BookConfig`
+        with normalized types (all IDs coerced to strings).
+
+        :param data: A dict that must contain at least "book_id".
+        :return: Normalized :class:`BookConfig` mapping.
+        :raises ValueError: If ``"book_id"`` is missing.
+        """
+        if "book_id" not in data:
+            raise ValueError("Missing required field 'book_id'")
+
+        out: BookConfig = {"book_id": str(data["book_id"])}
+
+        if "start_id" in data:
+            out["start_id"] = str(data["start_id"])
+        if "end_id" in data:
+            out["end_id"] = str(data["end_id"])
+        if "ignore_ids" in data:
+            with contextlib.suppress(Exception):
+                out["ignore_ids"] = [str(x) for x in data["ignore_ids"]]
+        return out
+
+    @staticmethod
+    def _dict_to_fontocr_cfg(data: dict[str, Any]) -> FontOCRConfig:
+        """
+        Convert a raw ``font_ocr`` dict into a :class:`FontOCRConfig`.
+        """
+        if not isinstance(data, dict):
+            return FontOCRConfig()
+
+        ishape = data.get("input_shape")
+        if isinstance(ishape, list):
+            ishape = tuple(ishape)  # [C, H, W] -> (C, H, W)
+
+        return FontOCRConfig(
+            model_name=data.get("model_name"),
+            model_dir=data.get("model_dir"),
+            input_shape=ishape,
+            device=data.get("device"),
+            precision=data.get("precision", "fp32"),
+            cpu_threads=data.get("cpu_threads", 10),
+            enable_hpi=data.get("enable_hpi", False),
+        )
+
+    @classmethod
+    def _dict_to_cleaner_cfg(cls, cfg: dict[str, Any]) -> TextCleanerConfig:
+        """
+        Convert a nested ``cleaner`` block into a
+        :class:`novel_downloader.models.TextCleanerConfig`.
+
+        :param cfg: configuration dictionary
+        :return: Aggregated title/content rules with external file contents merged
+        """
+        t_remove, t_replace = cls._merge_rules(cfg.get("title", {}) or {})
+        c_remove, c_replace = cls._merge_rules(cfg.get("content", {}) or {})
+        return TextCleanerConfig(
+            remove_invisible=cfg.get("remove_invisible", True),
+            title_remove_patterns=t_remove,
+            title_replacements=t_replace,
+            content_remove_patterns=c_remove,
+            content_replacements=c_replace,
+        )
+
+    @classmethod
+    def _merge_rules(cls, section: dict[str, Any]) -> tuple[list[str], dict[str, str]]:
+        """
+        Merge inline patterns/replacements with any enabled external files.
+
+        :param section: Mapping describing either the ``title`` or ``content`` rules.
+        :return: Tuple ``(remove_patterns, replace)`` after merging.
+        """
+        remove = list(section.get("remove_patterns") or [])
+        replace = dict(section.get("replace") or {})
+        ext = section.get("external") or {}
+        if ext.get("enabled", False):
+            rm_path = ext.get("remove_patterns") or ""
+            rp_path = ext.get("replace") or ""
+            remove += cls._load_str_list(rm_path)
+            replace.update(cls._load_str_dict(rp_path))
+        return remove, replace
+
+    @staticmethod
+    def _load_str_list(path: str) -> list[str]:
+        """
+        Load a JSON file containing a list of strings.
+
+        :param path: File path to a JSON array (e.g., ``["a", "b"]``).
+        :return: Parsed list on success; empty list if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return []
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+                return list(data) if isinstance(data, list) else []
+        except Exception:
+            return []
+
+    @staticmethod
+    def _load_str_dict(path: str) -> dict[str, str]:
+        """
+        Load a JSON file containing a dict of string-to-string mappings.
+
+        :param path: File path to a JSON object (e.g., ``{"old":"new"}``).
+        :return: Parsed dict on success; empty dict if ``path`` is empty, file is
+                 missing, or content is invalid.
+        """
+        if not path:
+            return {}
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+                return dict(data) if isinstance(data, dict) else {}
+        except Exception:
+            return {}

{novel_downloader-2.0.0 → novel_downloader-2.0.2}/src/novel_downloader/core/__init__.py

@@ -7,12 +7,11 @@ This package serves as the core layer of the novel_downloader system.
 
 It provides factory methods for constructing key components required for
 downloading and processing online novel content, including:
-
-- Downloader: Handles the full download lifecycle of a book or a batch of books.
-- Parser: Extracts structured data from HTML or SSR content.
-- Fetcher: Sends HTTP requests and manages sessions, including login if required.
-- Exporter: Responsible for exporting downloaded data into various output formats.
-- search: Provides unified search functionality across supported novel sites.
+  * Downloader: Handles the full download lifecycle of a book or a batch of books.
+  * Parser: Extracts structured data from HTML or SSR content.
+  * Fetcher: Sends HTTP requests and manages sessions, including login if required.
+  * Exporter: Responsible for exporting downloaded data into various output formats.
+  * search: Provides unified search functionality across supported novel sites.
 """
 
 __all__ = [