novel-downloader 1.4.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

novel_downloader/core/searchers/qianbi.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.qianbi
+-----------------------------------------
+
+"""
+
+import logging
+import re
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.core.searchers.registry import register_searcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+@register_searcher(
+    site_keys=["qianbi"],
+)
+class QianbiSearcher(BaseSearcher):
+    site_name = "qianbi"
+    priority = 3
+    SEARCH_URL = "https://www.23qb.com/search.html"
+
+    @classmethod
+    def _fetch_html(cls, keyword: str) -> str:
+        """
+        Fetch raw HTML from Qianbi's search page.
+
+        :param keyword: The search term to query on Qianbi.
+        :return: HTML text of the search results page, or an empty string on fail.
+        """
+        params = {"searchkey": keyword}
+        try:
+            response = cls._http_get(cls.SEARCH_URL, params=params)
+            return response.text
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+                exc_info=True,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        """
+        Parse raw HTML from Qianbi search results into list of SearchResult.
+
+        :param html_str: Raw HTML string from Qianbi search results page.
+        :param limit: Maximum number of results to return, or None for all.
+        :return: List of SearchResult dicts.
+        """
+        if html_str.find('<meta property="og:url"') != -1:
+            return cls._parse_detail_html(html_str)
+        return cls._parse_search_list_html(html_str, limit)
+
+    @classmethod
+    def _parse_detail_html(cls, html_str: str) -> list[SearchResult]:
+        """
+        Parse a single-book detail page, detected via <meta property="og:url">.
+
+        :param html_str: Raw HTML of the book detail page.
+        :return: A single-element list with the book's SearchResult.
+        """
+        doc = html.fromstring(html_str)
+        url = doc.xpath('//meta[@property="og:url"]/@content')
+        if not url:
+            return []
+
+        # extract book_id via regex
+        m = re.search(r"/book/(\d+)/", url[0])
+        book_id = m.group(1) if m else ""
+        # title from <h1 class="page-title">
+        title = (doc.xpath('//h1[@class="page-title"]/text()') or [""])[0].strip()
+        author = (doc.xpath('//a[contains(@href,"/author/")]/@title') or [""])[
+            0
+        ].strip()
+
+        return [
+            SearchResult(
+                site=cls.site_name,
+                book_id=book_id,
+                title=title,
+                author=author,
+                priority=cls.priority,
+            )
+        ]
+
+    @classmethod
+    def _parse_search_list_html(
+        cls, html_str: str, limit: int | None
+    ) -> list[SearchResult]:
+        """
+        Parse a multi-item search result page.
+
+        :param html_str: Raw HTML of the search-results page.
+        :param limit:    Maximum number of items to return, or None for all.
+        :return:         List of SearchResult.
+        """
+        doc = html.fromstring(html_str)
+        items = doc.xpath('//div[contains(@class,"module-search-item")]')
+        results: list[SearchResult] = []
+
+        for idx, item in enumerate(items):
+            if limit is not None and idx >= limit:
+                break
+            # Title and book_id
+            link = item.xpath('.//div[@class="novel-info-header"]/h3/a')[0]
+            title = link.text_content().strip()
+            href = link.get("href", "").strip("/")
+            book_id = href.replace("book/", "").strip("/")
+            # Author is not present on the page
+            author = ""
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    title=title,
+                    author=author,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/qidian.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.qidian
+--------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["qidian", "qd"],
+# )
+class QidianSearcher(BaseSearcher):
+    """
+    TODO: 现在默认没有 cookie 会跳转
+    """
+
+    site_name = "qidian"
+    priority = 0
+    SEARCH_URL = "https://www.qidian.com/so/{query}.html"
+
+    @classmethod
+    def _fetch_html(cls, keyword: str) -> str:
+        """
+        Fetch raw HTML from Qidian's search page.
+
+        :param keyword: The search term to query on Qidian.
+        :return: HTML text of the search results page, or an empty string on fail.
+        """
+        url = cls.SEARCH_URL.format(query=cls._quote(keyword))
+        try:
+            response = cls._http_get(url)
+            return response.text
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                url,
+                exc_info=True,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        """
+        Parse raw HTML from Qidian search results into list of SearchResult.
+
+        :param html_str: Raw HTML string from Qidian search results page.
+        :param limit: Maximum number of results to return, or None for all.
+        :return: List of SearchResult dicts.
+        """
+        doc = html.fromstring(html_str)
+        items = doc.xpath(
+            '//div[@id="result-list"]//li[contains(@class, "res-book-item")]'
+        )
+        results: list[SearchResult] = []
+
+        base_prio = getattr(cls, "priority", 0)
+        for idx, item in enumerate(items):
+            if limit is not None and idx >= limit:
+                break
+            book_id = item.get("data-bid")
+            title_elem = item.xpath('.//h3[@class="book-info-title"]/a')[0]
+            title = title_elem.text_content().strip()
+            author_nodes = item.xpath(
+                './/p[@class="author"]/a[@class="name"] | .//p[@class="author"]/i'
+            )
+            author = author_nodes[0].text_content().strip() if author_nodes else ""
+            prio = base_prio + idx
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    title=title,
+                    author=author,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/searchers/registry.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.searchers.registry
+----------------------------------------
+
+"""
+
+__all__ = ["register_searcher", "search"]
+
+from collections.abc import Callable, Sequence
+from typing import TypeVar
+
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+S = TypeVar("S", bound=BaseSearcher)
+
+_SEARCHER_REGISTRY: dict[str, type[BaseSearcher]] = {}
+
+
+def register_searcher(
+    site_keys: Sequence[str],
+) -> Callable[[type[S]], type[S]]:
+    """
+    Decorator to register a searcher class under given name.
+    """
+
+    def decorator(cls: type[S]) -> type[S]:
+        for key in site_keys:
+            _SEARCHER_REGISTRY[key] = cls
+        return cls
+
+    return decorator
+
+
+def search(
+    keyword: str,
+    sites: Sequence[str] | None = None,
+    limit: int | None = None,
+    per_site_limit: int = 5,
+) -> list[SearchResult]:
+    """
+    Perform a search for the given keyword across one or more registered sites,
+    then aggregate and sort the results by their `priority` value.
+
+    :param keyword: The search term or keyword to query.
+    :param sites:   An optional sequence of site keys to limit which searchers.
+    :param limit:   Maximum total number of results to return; if None, return all.
+    :param per_site_limit: Maximum number of search results per site.
+    :return:        A flat list of `SearchResult` objects.
+    """
+    keys = list(sites or _SEARCHER_REGISTRY.keys())
+    to_call = {_SEARCHER_REGISTRY[key] for key in keys if key in _SEARCHER_REGISTRY}
+
+    results: list[SearchResult] = []
+    for cls in to_call:
+        try:
+            results.extend(cls.search(keyword, limit=per_site_limit))
+        except Exception:
+            continue
+
+    results.sort(key=lambda res: res["priority"])
+    return results[:limit] if limit is not None else results

novel_downloader/locales/en.json

@@ -8,7 +8,6 @@
   "settings_help": "Configure downloader settings.",
   "settings_set_lang_help": "Switch language between Chinese and English.",
   "settings_set_config_help": "Set and save a custom YAML configuration file.",
-  "settings_update_rules_help": "Update site rules from a TOML/YAML/JSON file.",
   "settings_init_help": "Initialize default config and rule files in the current directory.",
   "settings_init_force_help": "Force overwrite if file already exists.",
   "settings_init_exists": "File already exists: {filename}",
@@ -20,8 +19,6 @@
   "settings_set_lang": "Language switched to {lang}",
   "settings_set_config": "Configuration file saved from {path}",
   "settings_set_config_fail": "Failed to save config file: {err}",
-  "settings_update_rules": "Site rules updated from {path}",
-  "settings_update_rules_fail": "Failed to update site rules: {err}",
   "settings_set_cookies_help": "Set cookies for a specific site.",
   "settings_set_cookies_prompt_site": "Site identifier (e.g. 'qidian')",
   "settings_set_cookies_prompt_payload": "Cookie payload (JSON or 'k=v; k2=v2')",
@@ -61,10 +58,12 @@
   "download_site_mode": "Mode: {mode}",
   "download_no_ids": "No book IDs provided. Exiting.",
   "download_fail_get_ids": "Failed to get book IDs from config: {err}",
+  "download_config_load_fail": "Failed to load config: {err}",
   "download_only_example": "Only example book IDs found (e.g. '{example}').",
   "download_edit_config": "Please edit your config and replace them with real book IDs.",
   "download_downloading": "Downloading book {book_id} from {site}...",
   "download_prompt_parse": "Parse...",
+  "download_login_failed": "Download login failed: please check your cookies or account credentials and try again.",
   "download_book_ids": "One or more book IDs to process",
   "download_option_start": "Start chapter ID (applies to the first book ID only)",
   "download_option_end": "End chapter ID (applies to the first book ID only)",
@@ -114,5 +113,14 @@
   "export_success_txt": "Successfully exported {book_id} as TXT.",
   "export_failed_txt": "Failed to export {book_id} as TXT: {err}",
   "export_success_epub": "Successfully exported {book_id} as EPUB.",
-  "export_failed_epub": "Failed to export {book_id} as EPUB: {err}"
+  "export_failed_epub": "Failed to export {book_id} as EPUB: {err}",
+
+  "help_search": "search for a book on one or more sites",
+  "help_search_sites": "which site keys to search (default: all)",
+  "help_search_keyword": "keyword to look for",
+  "help_search_limit": "Maximum number of search results",
+  "help_search_site_limit":  "Maximum number of search results per site",
+  "no_results": "No results found.",
+  "prompt_select_index": "Select a result by number (or press Enter to cancel): ",
+  "invalid_selection": "Invalid choice, please try again."
 }

novel_downloader/locales/zh.json

@@ -8,7 +8,6 @@
   "settings_help": "配置下载器设置",
   "settings_set_lang_help": "在中文和英文之间切换语言",
   "settings_set_config_help": "设置并保存自定义 YAML 配置文件",
-  "settings_update_rules_help": "从 TOML/YAML/JSON 文件更新站点规则",
   "settings_init_help": "在当前目录初始化默认配置和规则文件",
   "settings_init_force_help": "如果文件已存在则强制覆盖",
   "settings_init_exists": "文件已存在: {filename}",
@@ -20,8 +19,6 @@
   "settings_set_lang": "语言已切换为 {lang}",
   "settings_set_config": "已从 {path} 保存配置文件",
   "settings_set_config_fail": "保存配置文件失败: {err}",
-  "settings_update_rules": "已从 {path} 更新站点规则",
-  "settings_update_rules_fail": "更新站点规则失败: {err}",
   "settings_set_cookies_help": "为特定站点设置 Cookie",
   "settings_set_cookies_prompt_site": "站点标识 (例如 'qidian')",
   "settings_set_cookies_prompt_payload": "Cookie 内容 (JSON 或 'k=v; k2=v2')",
@@ -61,9 +58,11 @@
   "download_site_mode": "使用模式: {mode}",
   "download_no_ids": "未提供书籍 ID, 正在退出",
   "download_fail_get_ids": "从配置获取书籍 ID 失败: {err}",
+  "download_config_load_fail": "加载配置失败: {err}",
   "download_only_example": "只发现示例书籍 ID (例如 '{example}')",
   "download_edit_config": "请编辑配置并将示例 ID 替换为真实书籍 ID",
   "download_downloading": "正在从 {site} 下载书籍 {book_id}...",
+  "download_login_failed": "登录失败: 请检查您的 Cookie 或账户信息后重试",
   "download_prompt_parse": "结束...",
   "download_book_ids": "要处理的一个或多个小说 ID",
   "download_option_start": "起始章节 ID (仅用于第一个书籍 ID)",
@@ -114,5 +113,14 @@
   "export_success_txt": "成功将 {book_id} 导出为 TXT。",
   "export_failed_txt": "导出 {book_id} 为 TXT 失败: {err}",
   "export_success_epub": "成功将 {book_id} 导出为 EPUB",
-  "export_failed_epub": "导出 {book_id} 为 EPUB 失败: {err}"
+  "export_failed_epub": "导出 {book_id} 为 EPUB 失败: {err}",
+
+  "help_search": "在一个或多个站点搜索书籍",
+  "help_search_sites": "要搜索的站点键 (默认为全部)",
+  "help_search_keyword": "要搜索的关键字",
+  "help_search_limit": "总体搜索结果数量上限",
+  "help_search_site_limit": "单站点搜索结果数量上限",
+  "no_results": "未找到结果",
+  "prompt_select_index": "通过编号选择结果 (或按回车取消): ",
+  "invalid_selection": "无效选择, 请重试。"
 }

novel_downloader/models/__init__.py

@@ -5,7 +5,6 @@ novel_downloader.models
 
 """
 
-from .browser import NewContextOptions
 from .chapter import ChapterDict
 from .config import (
     BookConfig,
@@ -13,54 +12,29 @@ from .config import (
     ExporterConfig,
     FetcherConfig,
     ParserConfig,
+    TextCleanerConfig,
 )
 from .login import LoginField
-from .site_rules import (
-    BookInfoRules,
-    FieldRules,
-    RuleStep,
-    SiteProfile,
-    SiteRules,
-    SiteRulesDict,
-    VolumesRules,
-)
-from .tasks import (
-    CidTask,
-    HtmlTask,
-    RestoreTask,
-)
+from .search import SearchResult
 from .types import (
     BrowserType,
     LogLevel,
     ModeType,
-    SaveMode,
     SplitMode,
-    StorageBackend,
 )
 
 __all__ = [
-    "NewContextOptions",
     "BookConfig",
     "DownloaderConfig",
     "ParserConfig",
     "FetcherConfig",
     "ExporterConfig",
+    "TextCleanerConfig",
     "ChapterDict",
     "LoginField",
+    "SearchResult",
     "BrowserType",
     "ModeType",
-    "SaveMode",
-    "StorageBackend",
     "SplitMode",
     "LogLevel",
-    "BookInfoRules",
-    "FieldRules",
-    "RuleStep",
-    "SiteProfile",
-    "SiteRules",
-    "SiteRulesDict",
-    "VolumesRules",
-    "CidTask",
-    "HtmlTask",
-    "RestoreTask",
 ]

novel_downloader/models/config.py

@@ -16,14 +16,13 @@ These models are used to map loaded YAML or JSON config data into
 strongly typed Python objects for safer and cleaner access.
 """
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import NotRequired, TypedDict
 
 from .types import (
     BrowserType,
     ModeType,
     SplitMode,
-    StorageBackend,
 )
 
 
@@ -52,13 +51,11 @@ class DownloaderConfig:
     backoff_factor: float = 2.0
     raw_data_dir: str = "./raw_data"
     cache_dir: str = "./novel_cache"
-    download_workers: int = 4
-    parser_workers: int = 4
+    workers: int = 4
     skip_existing: bool = True
     login_required: bool = False
     save_html: bool = False
     mode: ModeType = "session"
-    storage_backend: StorageBackend = "json"
     storage_batch_size: int = 1
     username: str = ""
     password: str = ""
@@ -83,12 +80,20 @@ class ParserConfig:
     mode: ModeType = "session"
 
 
+@dataclass
+class TextCleanerConfig:
+    remove_invisible: bool = True
+    title_remove_patterns: list[str] = field(default_factory=list)
+    title_replacements: dict[str, str] = field(default_factory=dict)
+    content_remove_patterns: list[str] = field(default_factory=list)
+    content_replacements: dict[str, str] = field(default_factory=dict)
+
+
 @dataclass
 class ExporterConfig:
     cache_dir: str = "./novel_cache"
     raw_data_dir: str = "./raw_data"
     output_dir: str = "./downloads"
-    storage_backend: StorageBackend = "json"
     clean_text: bool = True
     make_txt: bool = True
     make_epub: bool = False
@@ -100,6 +105,7 @@ class ExporterConfig:
     include_toc: bool = False
     include_picture: bool = False
     split_mode: SplitMode = "book"
+    cleaner_cfg: TextCleanerConfig = field(default_factory=TextCleanerConfig)
 
 
 class BookConfig(TypedDict):

novel_downloader/models/search.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.models.search
+------------------------------
+
+"""
+
+from typing import TypedDict
+
+
+class SearchResult(TypedDict, total=True):
+    site: str
+    book_id: str
+    title: str
+    author: str
+    priority: int

novel_downloader/models/types.py

@@ -8,8 +8,6 @@ novel_downloader.models.types
 from typing import Literal
 
 ModeType = Literal["browser", "session"]
-SaveMode = Literal["overwrite", "skip"]
-StorageBackend = Literal["json", "sqlite"]
 SplitMode = Literal["book", "volume"]
 LogLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR"]
 BrowserType = Literal["chromium", "firefox", "webkit"]

novel_downloader/resources/config/settings.toml

@@ -21,11 +21,9 @@ request_interval = 2.0             # 同一本书各章节请求间隔 (秒)
 raw_data_dir = "./raw_data"        # 原始章节 JSON/DB 存放目录
 output_dir = "./downloads"         # 最终输出文件存放目录
 cache_dir = "./novel_cache"        # 本地缓存目录 (字体 / 图片等)
-download_workers = 2               # 并发下载线程数
-parser_workers = 2                 # 并发解析线程数
+workers = 2                        # 工作协程数
 skip_existing = true               # 是否跳过已存在章节
-storage_backend = "sqlite"         # 章节储存方法: json / sqlite
-storage_batch_size = 30            # SQLite 批量提交的章节数量
+storage_batch_size = 1             # SQLite 批量提交的章节数量
 
 [general.debug]
 save_html = false                  # 是否将抓取到的原始 HTML 保留到磁盘
@@ -129,3 +127,32 @@ filename_template = "{title}_{author}"  # 文件命名规则
 include_cover = true               # 是否在 EPUB 中包含封面
 include_toc = false                # 是否自动生成目录
 include_picture = false            # 是否下载章节图片 (体积较大)
+
+[cleaner]
+remove_invisible = true
+
+[cleaner.title]
+remove_patterns = [
+  '【[^】]*?】',
+  '[(（][^()（）]*?求票[^()（）]*?[)）]',
+]
+
+[cleaner.title.replace]
+'：' = ':'
+
+[cleaner.title.external]
+enabled = false
+remove_patterns = "path/to/title-remove.json"
+replace         = "path/to/title-replace.json"
+
+[cleaner.content]
+remove_patterns = []
+
+[cleaner.content.replace]
+'li子' = '例子'
+'pinbi词' = '屏蔽词'
+
+[cleaner.content.external]
+enabled = false
+remove_patterns = "path/to/content-remove.json"
+replace         = "path/to/content-replace.json"

novel_downloader/resources/css_styles/intro.css

@@ -0,0 +1,83 @@
+body {
+  font-family: serif;
+  line-height: 1.5;
+  height: 95%;
+  margin: 2em;
+}
+
+h1 {
+  font-size: 2em;
+  text-align: center;
+  margin-bottom: 1em;
+}
+
+p.new-page-after {
+  page-break-after: always;
+  margin: 0;
+  padding: 0;
+}
+
+.intro-info {
+  list-style: none;
+  padding: 0;
+  margin: 0 0 2em;
+}
+.intro-info li {
+  margin: 0.5em 0;
+  font-size: 1.1em;
+}
+
+.intro-summary {
+  margin-top: 1em;
+  font-size: 1em;
+}
+
+.vol-header {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  margin: 0;
+}
+
+.vol-border {
+  width: 100%;
+  text-align: center;
+  margin: 0 auto 0 auto;
+  text-indent: 0em;
+}
+.vol-border.flip {
+  transform: rotate(180deg);
+  -ms-transform: rotate(180deg);
+  -moz-transform: rotate(180deg);
+  -webkit-transform: rotate(180deg);
+  -o-transform: rotate(180deg);
+}
+
+.vol-title-main,
+.vol-title-sub {
+  font-weight: bold;
+  text-align: center;
+  text-indent: 0em;
+  color: #6e471c;
+  margin: 0.25em 0;
+  display: block;
+  color: #6e471c;
+  text-shadow: 1px 1px 2px rgba(255,255,255,0.8);
+}
+.vol-title-main {
+  font-size: 1.25rem;
+}
+.vol-title-sub {
+  font-size: 1rem;
+}
+
+.volume-intro-text {
+  margin: 1em 0;
+  padding: 0 1em;
+  text-align: justify;
+}
+.volume-intro-text p {
+  margin: 0.5em 0;
+  line-height: 1.6;
+}