novel-downloader 1.2.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

novel_downloader/config/loader.py

@@ -1,133 +1,160 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.config.loader
 --------------------------------
 
-Provides functionality to load YAML configuration files into Python
+Provides functionality to load Toml configuration files into Python
 dictionaries, with robust error handling and fallback support.
-
-This is typically used to load user-supplied or internal default config files.
 """
 
 import json
 import logging
-from importlib.abc import Traversable
-from importlib.resources import as_file
 from pathlib import Path
-from typing import Any, Dict, Optional, Union
-
-import yaml
+from typing import Any
 
 from novel_downloader.utils.cache import cached_load_config
-from novel_downloader.utils.constants import (
-    BASE_CONFIG_PATH,
-    SETTING_FILE,
-)
+from novel_downloader.utils.constants import SETTING_FILE
 
 logger = logging.getLogger(__name__)
 
 
-def resolve_config_path(
-    config_path: Optional[Union[str, Path]]
-) -> Optional[Union[Path, Traversable]]:
+def resolve_file_path(
+    user_path: str | Path | None,
+    local_filename: str | list[str],
+    fallback_path: Path,
+) -> Path | None:
     """
-    Resolve which configuration file to use, in this priority order:
+    Resolve the file path to use based on a prioritized lookup order.
 
-    1. User-specified path (the `config_path` argument).
-    2. `./settings.yaml` in the current working directory.
-    3. The global settings file (`SETTING_FILE`).
-    4. The internal default (`BASE_CONFIG_PATH`).
+    Priority:
+        1. A user-specified path (if provided and exists)
+        2. A file in the current working directory with the given name
+        3. A globally registered fallback path
 
-    Returns a Path to the first existing file, or None if none is found.
+    :param user_path: Optional user-specified file path.
+    :param local_filename: File name to check in the current working directory.
+    :param fallback_path: Fallback path used if other options are not available.
+    :return: A valid Path object if found, otherwise None.
     """
-    # 1. Try the user-provided path
-    if config_path:
-        path = Path(config_path).expanduser().resolve()
+    if user_path:
+        path = Path(user_path).expanduser().resolve()
         if path.is_file():
             return path
-        logger.warning("[config] Specified config file not found: %s", path)
+        logger.warning("[config] Specified file not found: %s", path)
 
-    # 2. Try ./settings.yaml in the current working directory
-    local_path = Path.cwd() / "settings.yaml"
-    if local_path.is_file():
-        logger.debug("[config] Using local settings.yaml at %s", local_path)
-        return local_path
+    filenames = [local_filename] if isinstance(local_filename, str) else local_filename
+    for name in filenames:
+        local_path = Path.cwd() / name
+        if local_path.is_file():
+            logger.debug("[config] Using local file: %s", local_path)
+            return local_path
 
-    # 3. Try the globally registered settings file
-    if SETTING_FILE.is_file():
-        logger.debug("[config] Using global settings file at %s", SETTING_FILE)
-        return SETTING_FILE
+    if fallback_path.is_file():
+        logger.debug("[config] Using fallback file: %s", fallback_path)
+        return fallback_path
 
-    # 4. Fallback to the internal default configuration
-    try:
-        logger.debug(
-            "[config] Falling back to internal base config at %s", BASE_CONFIG_PATH
-        )
-        return BASE_CONFIG_PATH
-    except Exception as e:
-        logger.error("[config] Failed to load internal base config: %s", e)
-        return None
+    logger.warning("[config] No file found at any location for: %s", local_filename)
+    return None
 
 
-@cached_load_config
-def load_config(config_path: Optional[Union[str, Path]]) -> Dict[str, Any]:
+def _validate_dict(data: Any, path: Path, format: str) -> dict[str, Any]:
     """
-    Load configuration data from a YAML file.
+    Validate that the parsed config is a dictionary.
 
-    :param config_path: Optional path to the YAML configuration file.
-    :return:            Parsed configuration as a dict.
+    :param data: The loaded content to validate.
+    :param path: Path to the original config file (used for logging).
+    :param format: Format name ('json', 'toml', etc.) for log context.
+    :return: The original data if valid, otherwise an empty dict.
     """
-    path = resolve_config_path(config_path)
-    if not path or not path.is_file():
-        logger.warning("[config] No valid config file found, using empty config.")
+    if not isinstance(data, dict):
+        logger.warning(
+            "[config] %s content is not a dictionary: %s",
+            format.upper(),
+            path,
+        )
         return {}
+    return data
 
-    with as_file(path) as real_path:
-        try:
-            content = real_path.read_text(encoding="utf-8")
-            ext = real_path.suffix.lower()
-        except Exception as e:
-            logger.error("[config] Failed to read config file '%s': %s", path, e)
-            return {}
 
-    data: Any = None
+def _load_by_extension(path: Path) -> dict[str, Any]:
+    """
+    Load a configuration file by its file extension.
 
+    Supports `.toml`, `.json`, and `.yaml`/`.yml` formats.
+
+    :param path: Path to the configuration file.
+    :return: Parsed configuration as a dictionary.
+    :raises ValueError: If the file extension is unsupported.
+    """
+    ext = path.suffix.lower()
     if ext == ".json":
+        with path.open("r", encoding="utf-8") as f:
+            data = json.load(f)
+            return _validate_dict(data, path, "json")
+
+    elif ext == ".toml":
+        import tomllib
+
+        with path.open("rb") as f:
+            data = tomllib.load(f)
+            return _validate_dict(data, path, "toml")
+
+    elif ext in {".yaml", ".yml"}:
         try:
-            data = json.loads(content)
-        except json.JSONDecodeError as e:
-            logger.error("[config] JSON parse error in '%s': %s", path, e)
-            return {}
+            import yaml
+        except ImportError as err:
+            raise ImportError(
+                "YAML config support requires PyYAML. "
+                "Install it via: pip install PyYAML"
+            ) from err
+        with path.open("r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+            return _validate_dict(data, path, "yaml")
+
     else:
-        try:
-            data = yaml.safe_load(content)
-        except yaml.YAMLError as e:
-            logger.error("[config] YAML parse error in '%s': %s", path, e)
-            return {}
+        raise ValueError(f"Unsupported config file extension: {ext}")
 
-    if data is None:
-        return {}
-    if not isinstance(data, dict):
-        logger.warning(
-            "[config] Expected dict in config file '%s', got %s",
-            path,
-            type(data).__name__,
-        )
-        return {}
 
-    return data
+@cached_load_config
+def load_config(
+    config_path: str | Path | None = None,
+) -> dict[str, Any]:
+    """
+    Load configuration data from a YAML file.
+
+    :param config_path: Optional path to the YAML configuration file.
+    :return:            Parsed configuration as a dict.
+    """
+    path = resolve_file_path(
+        user_path=config_path,
+        local_filename=[
+            "settings.toml",
+            "settings.yaml",
+            "settings.yml",
+            "settings.json",
+        ],
+        fallback_path=SETTING_FILE,
+    )
+
+    if not path or not path.is_file():
+        raise FileNotFoundError("No valid config file found.")
+
+    try:
+        return _load_by_extension(path)
+    except Exception as e:
+        logger.warning("[config] Failed to load config file: %s", e)
+    return {}
 
 
 def save_config_file(
-    source_path: Union[str, Path],
-    output_path: Union[str, Path] = SETTING_FILE,
+    source_path: str | Path,
+    output_path: str | Path = SETTING_FILE,
 ) -> None:
     """
-    Validate a YAML/JSON config file, load it into a dict,
+    Validate a TOML/YAML/JSON config file, load it into a dict,
     and then dump it as JSON to the internal SETTING_FILE.
 
-    :param source_path: The user-provided YAML file path.
+    :param source_path: The user-provided TOML file path.
     :param output_path: Destination path to save the config (default: SETTING_FILE).
     """
     source = Path(source_path).expanduser().resolve()
@@ -136,33 +163,14 @@ def save_config_file(
     if not source.is_file():
         raise FileNotFoundError(f"Source file not found: {source}")
 
-    ext = source.suffix.lower()
-
-    if ext in {".yaml", ".yml"}:
-        logger.debug("[config] Loading YAML for conversion: %s", source)
-        try:
-            with source.open("r", encoding="utf-8") as f:
-                data = yaml.safe_load(f)
-        except yaml.YAMLError as e:
-            logger.error("[config] Invalid YAML format: %s", e)
-            raise ValueError(f"Invalid YAML file: {source}") from e
-
-    elif ext == ".json":
-        logger.debug("[config] Loading JSON for saving: %s", source)
-        try:
-            with source.open("r", encoding="utf-8") as f:
-                data = json.load(f)
-        except json.JSONDecodeError as e:
-            logger.error("[config] Invalid JSON format: %s", e)
-            raise ValueError(f"Invalid JSON file: {source}") from e
-
-    else:
-        raise ValueError(f"Source file must be .yaml, .yml, or .json: {source}")
-
-    if not isinstance(data, dict):
-        raise ValueError(f"Config root must be a JSON/YAML object: {source}")
+    try:
+        data = _load_by_extension(source)
+    except (ValueError, ImportError) as e:
+        logger.error("[config] Failed to load config file: %s", e)
+        raise ValueError(f"Invalid config file: {source}") from e
 
     output.parent.mkdir(parents=True, exist_ok=True)
+
     try:
         with output.open("w", encoding="utf-8") as f:
             json.dump(data, f, indent=2, ensure_ascii=False)

novel_downloader/config/models.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.config.models
 ------------------------------
@@ -18,15 +17,17 @@ strongly typed Python objects for safer and cleaner access.
 """
 
 from dataclasses import dataclass
-from typing import Any, Dict, List, Literal, Optional, TypedDict
+from typing import Any, Literal, TypedDict
+
+ModeType = Literal["browser", "session", "async"]
+StorageBackend = Literal["json", "sqlite"]
 
 
 # === Requesters ===
 @dataclass
 class RequesterConfig:
-    wait_time: float = 5.0
     retry_times: int = 3
-    retry_interval: float = 5.0
+    backoff_factor: float = 2.0
     timeout: float = 30.0
     headless: bool = True
     user_data_folder: str = ""
@@ -34,8 +35,9 @@ class RequesterConfig:
     auto_close: bool = True
     disable_images: bool = True
     mute_audio: bool = True
-    mode: str = "session"  # browser / session / async
-    max_rps: Optional[float] = None  # Maximum requests per second
+    mode: ModeType = "session"
+    max_connections: int = 10
+    max_rps: float | None = None  # Maximum requests per second
 
 
 # === Downloaders ===
@@ -50,7 +52,9 @@ class DownloaderConfig:
     skip_existing: bool = True
     login_required: bool = False
     save_html: bool = False
-    mode: str = "session"  # browser / session / async
+    mode: ModeType = "session"
+    storage_backend: StorageBackend = "json"
+    storage_batch_size: int = 1
 
 
 # === Parsers ===
@@ -64,11 +68,11 @@ class ParserConfig:
     ocr_version: str = "v1.0"
     batch_size: int = 32
     gpu_mem: int = 500
-    gpu_id: Optional[int] = None
+    gpu_id: int | None = None
     ocr_weight: float = 0.6
     vec_weight: float = 0.4
     save_font_debug: bool = False
-    mode: str = "session"  # browser / session
+    mode: ModeType = "session"
 
 
 # === Savers ===
@@ -76,6 +80,7 @@ class ParserConfig:
 class SaverConfig:
     raw_data_dir: str = "./raw_data"
     output_dir: str = "./downloads"
+    storage_backend: StorageBackend = "json"
     clean_text: bool = True
     make_txt: bool = True
     make_epub: bool = False
@@ -85,6 +90,7 @@ class SaverConfig:
     filename_template: str = "{title}_{author}"
     include_cover: bool = True
     include_toc: bool = False
+    include_picture: bool = False
 
 
 class RuleStep(TypedDict, total=False):
@@ -105,39 +111,39 @@ class RuleStep(TypedDict, total=False):
     ]
 
     # —— BeautifulSoup 相关 —— #
-    selector: Optional[str]  # CSS 选择器, 用于 select/select_one/exclude
-    name: Optional[str]  # 标签名称, 用于 find/find_all
-    attrs: Optional[Dict[str, Any]]  # 属性过滤, 用于 find/find_all
-    limit: Optional[int]  # find_all 的最大匹配数
-    attr: Optional[str]  # 从元素获取属性值 (select/select_one/select_all)
+    selector: str | None  # CSS 选择器, 用于 select/select_one/exclude
+    name: str | None  # 标签名称, 用于 find/find_all
+    attrs: dict[str, Any] | None  # 属性过滤, 用于 find/find_all
+    limit: int | None  # find_all 的最大匹配数
+    attr: str | None  # 从元素获取属性值 (select/select_one/select_all)
 
     # —— 正则相关 —— #
-    pattern: Optional[str]  # 正则表达式
-    flags: Optional[int]  # re.I, re.M 等
-    group: Optional[int]  # 匹配结果中的第几个分组 (默认 0)
-    template: Optional[str]  # 自定义组合, 比如 "$1$2字"
+    pattern: str | None  # 正则表达式
+    flags: int | None  # re.I, re.M 等
+    group: int | None  # 匹配结果中的第几个分组 (默认 0)
+    template: str | None  # 自定义组合, 比如 "$1$2字"
 
     # —— 文本处理 —— #
-    chars: Optional[str]  # strip 要去除的字符集
-    old: Optional[str]  # replace 中要被替换的子串
-    new: Optional[str]  # replace 中新的子串
-    count: Optional[int]  # replace 中的最大替换次数
-    sep: Optional[str]  # split/join 的分隔符
-    index: Optional[int]  # split/select_all/select 之后取第几个元素
+    chars: str | None  # strip 要去除的字符集
+    old: str | None  # replace 中要被替换的子串
+    new: str | None  # replace 中新的子串
+    count: int | None  # replace 中的最大替换次数
+    sep: str | None  # split/join 的分隔符
+    index: int | None  # split/select_all/select 之后取第几个元素
 
 
 class FieldRules(TypedDict):
-    steps: List[RuleStep]
+    steps: list[RuleStep]
 
 
 class ChapterFieldRules(TypedDict):
     key: str
-    steps: List[RuleStep]
+    steps: list[RuleStep]
 
 
 class VolumesRulesOptional(TypedDict, total=False):
     volume_selector: str  # 有卷时选择 volume 块的 selector
-    volume_name_steps: List[RuleStep]
+    volume_name_steps: list[RuleStep]
     volume_mode: str  # Optional: "normal" (default) or "mixed"
     list_selector: str  # Optional: If "mixed" mode, parent container selector
 
@@ -145,7 +151,7 @@ class VolumesRulesOptional(TypedDict, total=False):
 class VolumesRules(VolumesRulesOptional):
     has_volume: bool  # 是否存在卷，false=未分卷
     chapter_selector: str  # 选择 chapter 节点的 selector
-    chapter_steps: List[ChapterFieldRules]  # 提取章节信息的步骤列表
+    chapter_steps: list[ChapterFieldRules]  # 提取章节信息的步骤列表
 
 
 class BookInfoRules(TypedDict, total=False):
@@ -175,4 +181,4 @@ class SiteRules(TypedDict):
     chapter: ChapterRules
 
 
-SiteRulesDict = Dict[str, SiteRules]
+SiteRulesDict = dict[str, SiteRules]

novel_downloader/config/site_rules.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.config.site_rules
 ----------------------------------
@@ -14,7 +13,6 @@ This module provides functionality to:
 import json
 import logging
 from pathlib import Path
-from typing import Union
 
 from novel_downloader.utils.cache import cached_load_config
 from novel_downloader.utils.constants import SITE_RULES_FILE
@@ -26,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 def save_rules_as_json(
-    source_path: Union[str, Path], output_path: Union[str, Path] = SITE_RULES_FILE
+    source_path: str | Path, output_path: str | Path = SITE_RULES_FILE
 ) -> None:
     """
     Load rules from source_path (toml, yaml, or json) and save as JSON.
@@ -78,7 +76,7 @@ def save_rules_as_json(
 
 
 @cached_load_config
-def load_site_rules(json_path: Union[str, Path] = SITE_RULES_FILE) -> SiteRulesDict:
+def load_site_rules(json_path: str | Path = SITE_RULES_FILE) -> SiteRulesDict:
     """
     Loads site scraping rules from a JSON file and caches the result for future access.
 

novel_downloader/core/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core
 ---------------------

novel_downloader/core/downloaders/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.downloaders
 ---------------------------------
@@ -11,11 +10,12 @@ Each downloader is responsible for orchestrating the full lifecycle
 of retrieving, parsing, and saving novel content for a given source.
 """
 
-from .common_asynb_downloader import CommonAsyncDownloader
-from .common_downloader import CommonDownloader
-from .qidian_downloader import QidianDownloader
+from .biquge import BiqugeDownloader
+from .common import CommonAsyncDownloader, CommonDownloader
+from .qidian import QidianDownloader
 
 __all__ = [
+    "BiqugeDownloader",
     "CommonAsyncDownloader",
     "CommonDownloader",
     "QidianDownloader",

novel_downloader/core/downloaders/base/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.downloaders.base
+--------------------------------------
+
+"""
+
+from .base_async import BaseAsyncDownloader
+from .base_sync import BaseDownloader
+
+__all__ = [
+    "BaseAsyncDownloader",
+    "BaseDownloader",
+]

novel_downloader/core/downloaders/{base_async_downloader.py → base/base_async.py}

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.downloaders.base_async_downloader
--------------------------------------------------------
+novel_downloader.core.downloaders.base.base_async
+-------------------------------------------------
 
 Defines the abstract base class `BaseAsyncDownloader`, which provides a
 common interface and reusable logic for all downloader implementations.
@@ -11,19 +10,14 @@ common interface and reusable logic for all downloader implementations.
 import abc
 import logging
 from pathlib import Path
-from typing import List
 
 from novel_downloader.config import DownloaderConfig
 from novel_downloader.core.interfaces import (
+    AsyncDownloaderProtocol,
     AsyncRequesterProtocol,
     ParserProtocol,
     SaverProtocol,
 )
-from novel_downloader.core.interfaces.async_downloader_protocol import (
-    AsyncDownloaderProtocol,
-)
-
-logger = logging.getLogger(__name__)
 
 
 class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
@@ -53,6 +47,51 @@ class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
         self._raw_data_dir.mkdir(parents=True, exist_ok=True)
         self._cache_dir.mkdir(parents=True, exist_ok=True)
 
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+
+    async def download(self, book_ids: list[str]) -> None:
+        """
+        The general batch download process:
+          1. Iterate over all book IDs
+          2. For each ID, call `download_one()`
+
+        :param book_ids: A list of book identifiers to download.
+        """
+        await self.prepare()
+
+        # 2) batch download
+        for idx, book_id in enumerate(book_ids, start=1):
+            self.logger.debug(
+                "[%s] Starting download for %r (%s/%s)",
+                self.__class__.__name__,
+                book_id,
+                idx,
+                len(book_ids),
+            )
+            try:
+                await self.download_one(book_id)
+            except Exception as e:
+                self._handle_download_exception(book_id, e)
+
+    @abc.abstractmethod
+    async def download_one(self, book_id: str) -> None:
+        """
+        The full download logic for a single book.
+
+        Subclasses must implement this method.
+
+        :param book_id: The identifier of the book to download.
+        """
+        ...
+
+    async def prepare(self) -> None:
+        """
+        Optional hook called before downloading each book.
+
+        Subclasses can override this method to perform pre-download setup.
+        """
+        return
+
     @property
     def requester(self) -> AsyncRequesterProtocol:
         return self._requester
@@ -97,49 +136,6 @@ class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
     def request_interval(self) -> float:
         return self._config.request_interval
 
-    async def prepare(self) -> None:
-        """
-        Optional hook called before downloading each book.
-
-        Subclasses can override this method to perform pre-download setup.
-        """
-        return
-
-    async def download(self, book_ids: List[str]) -> None:
-        """
-        The general batch download process:
-          1. Iterate over all book IDs
-          2. For each ID, call `download_one()`
-
-        :param book_ids: A list of book identifiers to download.
-        """
-        await self.prepare()
-
-        # 2) batch download
-        for idx, book_id in enumerate(book_ids, start=1):
-            logger.debug(
-                "[%s] Starting download for %r (%s/%s)",
-                self.__class__.__name__,
-                book_id,
-                idx,
-                len(book_ids),
-            )
-            try:
-                await self.download_one(book_id)
-            except Exception as e:
-                self._handle_download_exception(book_id, e)
-
-    @abc.abstractmethod
-    async def download_one(self, book_id: str) -> None:
-        """
-        The full download logic for a single book.
-
-        Subclasses must implement this method.
-
-        :param book_id: The identifier of the book to download.
-        """
-        ...
-
     def _handle_download_exception(self, book_id: str, error: Exception) -> None:
         """
         Handle download errors in a consistent way.
@@ -149,7 +145,7 @@ class BaseAsyncDownloader(AsyncDownloaderProtocol, abc.ABC):
         :param book_id: The ID of the book that failed.
         :param error: The exception raised during download.
         """
-        logger.warning(
+        self.logger.warning(
             "[%s] Failed to download %r: %s",
             self.__class__.__name__,
             book_id,