novel-downloader 1.3.3__py3-none-any.whl → 1.4.0__py3-none-any.whl

novel_downloader/core/parsers/sfacg/main_parser.py

@@ -7,10 +7,10 @@ novel_downloader.core.parsers.sfacg.main_parser
 
 from typing import Any
 
-from lxml import etree
+from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 
 
 class SfacgParser(BaseParser):
@@ -40,20 +40,20 @@ class SfacgParser(BaseParser):
 
     def parse_book_info(
         self,
-        html_str: list[str],
+        html_list: list[str],
         **kwargs: Any,
     ) -> dict[str, Any]:
         """
         Parse a book info page and extract metadata and chapter structure.
 
-        :param html: Raw HTML of the book info page.
+        :param html_list: Raw HTML of the book info page.
         :return: Parsed metadata and chapter structure as a dictionary.
         """
-        if len(html_str) < 2:
+        if len(html_list) < 2:
             return {}
 
-        info_tree = etree.HTML(html_str[0])
-        catalog_tree = etree.HTML(html_str[1])
+        info_tree = html.fromstring(html_list[0])
+        catalog_tree = html.fromstring(html_list[1])
 
         result: dict[str, Any] = {}
 
@@ -113,25 +113,25 @@ class SfacgParser(BaseParser):
 
     def parse_chapter(
         self,
-        html_str: list[str],
+        html_list: list[str],
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
         Parse a single chapter page and extract clean text or simplified HTML.
 
-        :param html: Raw HTML of the chapter page.
+        :param html_list: Raw HTML of the chapter page.
         :param chapter_id: Identifier of the chapter being parsed.
         :return: Cleaned chapter content as plain text or minimal HTML.
         """
-        if not html_str:
+        if not html_list:
             return None
         keywords = [
             "本章为VIP章节",  # 本章为VIP章节，订阅后可立即阅读
         ]
-        if any(kw in html_str[0] for kw in keywords):
+        if any(kw in html_list[0] for kw in keywords):
             return None
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_list[0])
 
         content_lines: list[str] = []
         content_nodes = tree.xpath(self._CHAPTER_CONTENT_NODES_XPATH)

novel_downloader/core/parsers/yamibo/main_parser.py

@@ -7,10 +7,10 @@ novel_downloader.core.parsers.yamibo.main_parser
 
 from typing import Any
 
-from lxml import etree
+from lxml import html
 
 from novel_downloader.core.parsers.base import BaseParser
-from novel_downloader.utils.chapter_storage import ChapterDict
+from novel_downloader.models import ChapterDict
 
 
 class YamiboParser(BaseParser):
@@ -49,19 +49,19 @@ class YamiboParser(BaseParser):
 
     def parse_book_info(
         self,
-        html_str: list[str],
+        html_list: list[str],
         **kwargs: Any,
     ) -> dict[str, Any]:
         """
         Parse a book info page and extract metadata and chapter structure.
 
-        :param html: Raw HTML of the book info page.
+        :param html_list: Raw HTML of the book info page.
         :return: Parsed metadata and chapter structure as a dictionary.
         """
-        if not html_str:
+        if not html_list:
             return {}
 
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_list[0])
         result: dict[str, Any] = {}
 
         result["book_name"] = tree.xpath(self._BOOK_NAME_XPATH).strip()
@@ -151,20 +151,20 @@ class YamiboParser(BaseParser):
 
     def parse_chapter(
         self,
-        html_str: list[str],
+        html_list: list[str],
         chapter_id: str,
         **kwargs: Any,
     ) -> ChapterDict | None:
         """
         Parse a single chapter page and extract clean text or simplified HTML.
 
-        :param html: Raw HTML of the chapter page.
+        :param html_list: Raw HTML of the chapter page.
         :param chapter_id: Identifier of the chapter being parsed.
         :return: Cleaned chapter content as plain text or minimal HTML.
         """
-        if not html_str:
+        if not html_list:
             return None
-        tree = etree.HTML(html_str[0])
+        tree = html.fromstring(html_list[0])
 
         content_lines = tree.xpath(self._CHAPTER_CONTENT_XPATH)
         content = "\n\n".join(line.strip() for line in content_lines if line.strip())

novel_downloader/locales/en.json

@@ -3,7 +3,7 @@
   "help_config": "Path to config file",
   "help_download": "Download novels",
   "help_clean": "Clean cache and configuration files",
-  "main_no_command": "No command provided, entering interactive mode...",
+  "help_export": "Export downloaded novels",
 
   "settings_help": "Configure downloader settings.",
   "settings_set_lang_help": "Switch language between Chinese and English.",
@@ -65,6 +65,15 @@
   "download_edit_config": "Please edit your config and replace them with real book IDs.",
   "download_downloading": "Downloading book {book_id} from {site}...",
   "download_prompt_parse": "Parse...",
+  "download_book_ids": "One or more book IDs to process",
+  "login_description": "Description",
+  "login_hint": "Hint",
+  "login_manual_prompt": ">> Please complete login in your browser and press Enter to continue...",
+  "login_use_config": "Using value from config.",
+  "login_enter_password": "Enter password: ",
+  "login_enter_cookie": "Enter cookies: ",
+  "login_enter_value": "Enter value: ",
+  "login_required_field": "This field is required. Please enter a value.",
 
   "login_prompt_intro": "Manual login is required. Please switch to the browser and log in.",
   "login_prompt_press_enter": "Attempt {attempt}/{max_retries}: Press Enter after completing login in the browser...",
@@ -96,5 +105,12 @@
   "clean_nothing": "No clean option specified",
   "clean_deleted": "Deleted",
   "clean_not_found": "Not found",
-  "clean_cancelled": "Clean operation cancelled"
+  "clean_cancelled": "Clean operation cancelled",
+
+  "export_format_help": "Export format: txt, epub, or all",
+  "export_processing": "Exporting book {book_id} in format: {format}",
+  "export_success_txt": "Successfully exported {book_id} as TXT.",
+  "export_failed_txt": "Failed to export {book_id} as TXT: {err}",
+  "export_success_epub": "Successfully exported {book_id} as EPUB.",
+  "export_failed_epub": "Failed to export {book_id} as EPUB: {err}"
 }

novel_downloader/locales/zh.json

@@ -3,7 +3,7 @@
   "help_config": "配置文件路径",
   "help_download": "下载小说",
   "help_clean": "清理缓存和配置文件",
-  "main_no_command": "未提供命令, 进入交互模式...",
+  "help_export": "导出已下载的小说",
 
   "settings_help": "配置下载器设置",
   "settings_set_lang_help": "在中文和英文之间切换语言",
@@ -65,6 +65,15 @@
   "download_edit_config": "请编辑配置并将示例 ID 替换为真实书籍 ID",
   "download_downloading": "正在从 {site} 下载书籍 {book_id}...",
   "download_prompt_parse": "结束...",
+  "download_book_ids": "要处理的一个或多个小说 ID",
+  "login_description": "说明",
+  "login_hint": "提示",
+  "login_manual_prompt": ">> 请在浏览器中完成登录后按回车继续...",
+  "login_use_config": "使用配置中的默认值",
+  "login_enter_password": "请输入密码: ",
+  "login_enter_cookie": "请输入 Cookie: ",
+  "login_enter_value": "请输入值: ",
+  "login_required_field": "该字段是必填项, 请重新输入",
 
   "login_prompt_intro": "需要手动登录, 请切换到浏览器窗口完成登录",
   "login_prompt_press_enter": "第 {attempt}/{max_retries} 次尝试: 请在浏览器中完成登录后按回车键...",
@@ -96,5 +105,12 @@
   "clean_nothing": "未指定任何要清理的内容",
   "clean_deleted": "已删除",
   "clean_not_found": "未找到",
-  "clean_cancelled": "已取消清理操作"
+  "clean_cancelled": "已取消清理操作",
+
+  "export_format_help": "导出格式: txt、epub 或 all",
+  "export_processing": "正在导出小说 {book_id}, 格式: {format}",
+  "export_success_txt": "成功将 {book_id} 导出为 TXT。",
+  "export_failed_txt": "导出 {book_id} 为 TXT 失败: {err}",
+  "export_success_epub": "成功将 {book_id} 导出为 EPUB",
+  "export_failed_epub": "导出 {book_id} 为 EPUB 失败: {err}"
 }

novel_downloader/models/__init__.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models
+-----------------------
+
+"""
+
+from .browser import NewContextOptions
+from .chapter import ChapterDict
+from .config import (
+    DownloaderConfig,
+    ExporterConfig,
+    FetcherConfig,
+    ParserConfig,
+)
+from .login import LoginField
+from .site_rules import (
+    BookInfoRules,
+    FieldRules,
+    RuleStep,
+    SiteProfile,
+    SiteRules,
+    SiteRulesDict,
+    VolumesRules,
+)
+from .tasks import (
+    CidTask,
+    HtmlTask,
+    RestoreTask,
+)
+from .types import (
+    BrowserType,
+    LogLevel,
+    ModeType,
+    SaveMode,
+    SplitMode,
+    StorageBackend,
+)
+
+__all__ = [
+    "NewContextOptions",
+    "DownloaderConfig",
+    "ParserConfig",
+    "FetcherConfig",
+    "ExporterConfig",
+    "ChapterDict",
+    "LoginField",
+    "BrowserType",
+    "ModeType",
+    "SaveMode",
+    "StorageBackend",
+    "SplitMode",
+    "LogLevel",
+    "BookInfoRules",
+    "FieldRules",
+    "RuleStep",
+    "SiteProfile",
+    "SiteRules",
+    "SiteRulesDict",
+    "VolumesRules",
+    "CidTask",
+    "HtmlTask",
+    "RestoreTask",
+]

novel_downloader/models/browser.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.browser
+-------------------------------
+
+"""
+
+from pathlib import Path
+from typing import TypedDict
+
+from playwright.async_api import ViewportSize
+
+
+class NewContextOptions(TypedDict, total=False):
+    user_agent: str
+    locale: str
+    storage_state: Path
+    viewport: ViewportSize
+    java_script_enabled: bool
+    ignore_https_errors: bool
+    extra_http_headers: dict[str, str]

novel_downloader/models/chapter.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.chapter
+-------------------------------
+
+"""
+
+from typing import Any, TypedDict
+
+
+class ChapterDict(TypedDict, total=True):
+    """
+    TypedDict for a novel chapter.
+
+    Fields:
+        id      -- Unique chapter identifier
+        title   -- Chapter title
+        content -- Chapter text
+        extra   -- Arbitrary metadata (e.g. author remarks, timestamps)
+    """
+
+    id: str
+    title: str
+    content: str
+    extra: dict[str, Any]

novel_downloader/models/config.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.config
+------------------------------
+
+Defines structured configuration models using dataclasses for each
+major component in the novel_downloader pipeline.
+
+Each config section corresponds to a specific stage of the pipeline:
+- RequesterConfig: network settings for requests and DrissionPage
+- DownloaderConfig: chapter download behavior and local raw data paths
+- ParserConfig: font decoding, cache handling, and debug options
+- SaverConfig: output formatting, export formats, and filename templates
+
+These models are used to map loaded YAML or JSON config data into
+strongly typed Python objects for safer and cleaner access.
+"""
+
+from dataclasses import dataclass
+
+from .types import (
+    BrowserType,
+    ModeType,
+    SplitMode,
+    StorageBackend,
+)
+
+
+@dataclass
+class FetcherConfig:
+    request_interval: float = 2.0
+    retry_times: int = 3
+    backoff_factor: float = 2.0
+    timeout: float = 30.0
+    headless: bool = False
+    disable_images: bool = False
+    mode: ModeType = "session"
+    max_connections: int = 10
+    max_rps: float | None = None  # Maximum requests per second
+    proxy: str | None = None
+    user_agent: str | None = None
+    headers: dict[str, str] | None = None
+    browser_type: BrowserType = "chromium"
+    verify_ssl: bool = True
+
+
+@dataclass
+class DownloaderConfig:
+    request_interval: float = 2.0
+    retry_times: int = 3
+    backoff_factor: float = 2.0
+    raw_data_dir: str = "./raw_data"
+    cache_dir: str = "./novel_cache"
+    download_workers: int = 4
+    parser_workers: int = 4
+    skip_existing: bool = True
+    login_required: bool = False
+    save_html: bool = False
+    mode: ModeType = "session"
+    storage_backend: StorageBackend = "json"
+    storage_batch_size: int = 1
+    username: str = ""
+    password: str = ""
+    cookies: str = ""
+
+
+@dataclass
+class ParserConfig:
+    cache_dir: str = "./novel_cache"
+    decode_font: bool = False
+    use_freq: bool = False
+    use_ocr: bool = True
+    use_vec: bool = False
+    ocr_version: str = "v1.0"
+    batch_size: int = 32
+    gpu_mem: int = 500
+    gpu_id: int | None = None
+    ocr_weight: float = 0.6
+    vec_weight: float = 0.4
+    save_font_debug: bool = False
+    mode: ModeType = "session"
+
+
+@dataclass
+class ExporterConfig:
+    cache_dir: str = "./novel_cache"
+    raw_data_dir: str = "./raw_data"
+    output_dir: str = "./downloads"
+    storage_backend: StorageBackend = "json"
+    clean_text: bool = True
+    make_txt: bool = True
+    make_epub: bool = False
+    make_md: bool = False
+    make_pdf: bool = False
+    append_timestamp: bool = True
+    filename_template: str = "{title}_{author}"
+    include_cover: bool = True
+    include_toc: bool = False
+    include_picture: bool = False
+    split_mode: SplitMode = "book"

novel_downloader/models/login.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.login
+-----------------------------
+
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass
+class LoginField:
+    name: str
+    label: str
+    type: Literal["text", "password", "cookie", "manual_login"]
+    required: bool
+    default: str = ""
+    placeholder: str = ""
+    description: str = ""

novel_downloader/models/site_rules.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.site_rules
+----------------------------------
+
+"""
+
+from typing import Any, Literal, TypedDict
+
+
+class RuleStep(TypedDict, total=False):
+    # —— 操作类型 —— #
+    type: Literal[
+        "attr",
+        "select_one",
+        "select",
+        "find",
+        "find_all",
+        "exclude",
+        "regex",
+        "text",
+        "strip",
+        "replace",
+        "split",
+        "join",
+    ]
+
+    # —— BeautifulSoup 相关 —— #
+    selector: str | None  # CSS 选择器, 用于 select/select_one/exclude
+    name: str | None  # 标签名称, 用于 find/find_all
+    attrs: dict[str, Any] | None  # 属性过滤, 用于 find/find_all
+    limit: int | None  # find_all 的最大匹配数
+    attr: str | None  # 从元素获取属性值 (select/select_one/select_all)
+
+    # —— 正则相关 —— #
+    pattern: str | None  # 正则表达式
+    flags: int | None  # re.I, re.M 等
+    group: int | None  # 匹配结果中的第几个分组 (默认 0)
+    template: str | None  # 自定义组合, 比如 "$1$2字"
+
+    # —— 文本处理 —— #
+    chars: str | None  # strip 要去除的字符集
+    old: str | None  # replace 中要被替换的子串
+    new: str | None  # replace 中新的子串
+    count: int | None  # replace 中的最大替换次数
+    sep: str | None  # split/join 的分隔符
+    index: int | None  # split/select_all/select 之后取第几个元素
+
+
+class FieldRules(TypedDict):
+    steps: list[RuleStep]
+
+
+class ChapterFieldRules(TypedDict):
+    key: str
+    steps: list[RuleStep]
+
+
+class VolumesRulesOptional(TypedDict, total=False):
+    volume_selector: str  # 有卷时选择 volume 块的 selector
+    volume_name_steps: list[RuleStep]
+    volume_mode: str  # Optional: "normal" (default) or "mixed"
+    list_selector: str  # Optional: If "mixed" mode, parent container selector
+
+
+class VolumesRules(VolumesRulesOptional):
+    has_volume: bool  # 是否存在卷，false=未分卷
+    chapter_selector: str  # 选择 chapter 节点的 selector
+    chapter_steps: list[ChapterFieldRules]  # 提取章节信息的步骤列表
+
+
+class BookInfoRules(TypedDict, total=False):
+    book_name: FieldRules
+    author: FieldRules
+    cover_url: FieldRules
+    update_time: FieldRules
+    serial_status: FieldRules
+    word_count: FieldRules
+    summary: FieldRules
+    volumes: VolumesRules
+
+
+class ChapterRules(TypedDict, total=False):
+    title: FieldRules
+    content: FieldRules
+
+
+class SiteProfile(TypedDict):
+    book_info_url: str
+    chapter_url: str
+
+
+class SiteRules(TypedDict):
+    profile: SiteProfile
+    book_info: BookInfoRules
+    chapter: ChapterRules
+
+
+SiteRulesDict = dict[str, SiteRules]

novel_downloader/models/tasks.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.tasks
+-----------------------------
+
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass
+class CidTask:
+    prev_cid: str | None
+    cid: str
+    retry: int = 0
+    vol_idx: int = 0
+    chap_idx: int = 0
+
+
+@dataclass
+class RestoreTask:
+    vol_idx: int
+    chap_idx: int
+    prev_cid: str
+
+
+@dataclass
+class HtmlTask:
+    cid: str
+    retry: int
+    html_list: list[str]
+    vol_idx: int = 0
+    chap_idx: int = 0

novel_downloader/models/types.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.types
+-----------------------------
+
+"""
+
+from typing import Literal
+
+ModeType = Literal["browser", "session"]
+SaveMode = Literal["overwrite", "skip"]
+StorageBackend = Literal["json", "sqlite"]
+SplitMode = Literal["book", "volume"]
+LogLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR"]
+BrowserType = Literal["chromium", "firefox", "webkit"]