novel-downloader 1.5.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

novel_downloader/config/{loader.py → file_io.py}

@@ -1,26 +1,23 @@
 #!/usr/bin/env python3
 """
-novel_downloader.config.loader
-------------------------------
+novel_downloader.config.file_io
+-------------------------------
 
-Provides functionality to load Toml configuration files into Python
-dictionaries, with robust error handling and fallback support.
+Provides functionality to load Toml configuration files into Python dict
 """
 
-__all__ = ["load_config"]
-
 import json
 import logging
 from pathlib import Path
-from typing import Any
+from typing import Any, TypeVar
 
-from novel_downloader.utils.cache import cached_load_config
 from novel_downloader.utils.constants import SETTING_FILE
 
+T = TypeVar("T")
 logger = logging.getLogger(__name__)
 
 
-def resolve_file_path(
+def _resolve_file_path(
     user_path: str | Path | None,
     local_filename: str | list[str],
     fallback_path: Path,
@@ -29,9 +26,9 @@ def resolve_file_path(
     Resolve the file path to use based on a prioritized lookup order.
 
     Priority:
-        1. A user-specified path (if provided and exists)
-        2. A file in the current working directory with the given name
-        3. A globally registered fallback path
+      1. A user-specified path (if provided and exists)
+      2. A file in the current working directory with the given name
+      3. A globally registered fallback path
 
     :param user_path: Optional user-specified file path.
     :param local_filename: File name to check in the current working directory.
@@ -117,7 +114,6 @@ def _load_by_extension(path: Path) -> dict[str, Any]:
         raise ValueError(f"Unsupported config file extension: {ext}")
 
 
-@cached_load_config
 def load_config(
     config_path: str | Path | None = None,
 ) -> dict[str, Any]:
@@ -125,9 +121,9 @@ def load_config(
     Load configuration data from a Toml file.
 
     :param config_path: Optional path to the Toml configuration file.
-    :return:            Parsed configuration as a dict.
+    :return: Parsed configuration as a dict.
     """
-    path = resolve_file_path(
+    path = _resolve_file_path(
         user_path=config_path,
         local_filename=[
             "settings.toml",
@@ -148,6 +144,46 @@ def load_config(
     return {}
 
 
+def get_config_value(keys: list[str], default: T) -> T:
+    """
+    Safely retrieve a nested config value.
+    """
+    cur = load_config()
+    for i, k in enumerate(keys):
+        if not isinstance(cur, dict):
+            return default
+        if i == len(keys) - 1:
+            val = cur.get(k, default)
+            return val if isinstance(val, type(default)) else default
+        cur = cur.get(k, {})
+    return default
+
+
+def save_config(
+    config: dict[str, Any],
+    output_path: str | Path = SETTING_FILE,
+) -> None:
+    """
+    Save configuration data to disk in JSON format.
+
+    :param config: Dictionary containing configuration data to save.
+    :param output_path: Destination path to save the config (default: SETTING_FILE).
+    :raises Exception: If writing to the file fails.
+    """
+    output = Path(output_path).expanduser().resolve()
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        with output.open("w", encoding="utf-8") as f:
+            json.dump(config, f, indent=2, ensure_ascii=False)
+    except Exception as e:
+        logger.error("[config] Failed to write config JSON '%s': %s", output, e)
+        raise
+
+    logger.info("[config] Configuration successfully saved to JSON: %s", output)
+    return
+
+
 def save_config_file(
     source_path: str | Path,
     output_path: str | Path = SETTING_FILE,
@@ -158,9 +194,9 @@ def save_config_file(
 
     :param source_path: The user-provided TOML file path.
     :param output_path: Destination path to save the config (default: SETTING_FILE).
+    :raises Exception: If writing to the file fails.
     """
     source = Path(source_path).expanduser().resolve()
-    output = Path(output_path).expanduser().resolve()
 
     if not source.is_file():
         raise FileNotFoundError(f"Source file not found: {source}")
@@ -171,14 +207,5 @@ def save_config_file(
         logger.error("[config] Failed to load config file: %s", e)
         raise ValueError(f"Invalid config file: {source}") from e
 
-    output.parent.mkdir(parents=True, exist_ok=True)
-
-    try:
-        with output.open("w", encoding="utf-8") as f:
-            json.dump(data, f, indent=2, ensure_ascii=False)
-    except Exception as e:
-        logger.error("[config] Failed to write config JSON '%s': %s", output, e)
-        raise
-
-    logger.info("[config] Configuration successfully saved to JSON: %s", output)
+    save_config(data, output_path)
     return

novel_downloader/core/__init__.py

@@ -7,11 +7,11 @@ This package serves as the core layer of the novel_downloader system.
 
 It provides factory methods for constructing key components required for
 downloading and processing online novel content, including:
-
-- Downloader: Handles the full download lifecycle of a book or a batch of books.
-- Parser: Extracts structured data from HTML or SSR content.
-- Fetcher: Sends HTTP requests and manages sessions, including login if required.
-- Exporter: Responsible for exporting downloaded data into various output formats.
+  * Downloader: Handles the full download lifecycle of a book or a batch of books.
+  * Parser: Extracts structured data from HTML or SSR content.
+  * Fetcher: Sends HTTP requests and manages sessions, including login if required.
+  * Exporter: Responsible for exporting downloaded data into various output formats.
+  * search: Provides unified search functionality across supported novel sites.
 """
 
 __all__ = [

novel_downloader/core/archived/deqixs/fetcher.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.fetcher
+---------------------------------------------
+
+"""
+
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.models import FetcherConfig
+from novel_downloader.utils import async_jitter_sleep
+
+# from novel_downloader.core.fetchers.registry import register_fetcher
+
+
+# @register_fetcher(
+#     site_keys=["deqixs"],
+# )
+class DeqixsSession(BaseSession):
+    """
+    A session class for interacting with the 得奇小说网 (www.deqixs.com) novel website.
+    """
+
+    BASE_URL = "https://www.deqixs.com"
+    BOOK_INFO_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/"
+    CHAPTER_URL = "https://www.deqixs.com/xiaoshuo/{book_id}/{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("deqixs", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as a string.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The chapter content as a string.
+        """
+        html_pages: list[str] = []
+        idx = 1
+
+        while True:
+            chapter_suffix = chapter_id if idx == 1 else f"{chapter_id}-{idx}"
+            relative_path = f"/xiaoshuo/{book_id}/{chapter_suffix}.html"
+            full_url = self.BASE_URL + relative_path
+
+            if idx > 1 and relative_path not in html_pages[-1]:
+                break
+
+            try:
+                html = await self.fetch(full_url, **kwargs)
+            except Exception as exc:
+                self.logger.warning(
+                    "[async] get_book_chapter(%s page %d) failed: %s",
+                    chapter_id,
+                    idx,
+                    exc,
+                )
+                break
+
+            html_pages.append(html)
+            idx += 1
+            await async_jitter_sleep(
+                self.request_interval,
+                mul_spread=1.1,
+                max_sleep=self.request_interval + 2,
+            )
+
+        return html_pages
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)

novel_downloader/core/archived/deqixs/parser.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.parser
+--------------------------------------------
+
+"""
+
+from typing import Any
+
+from lxml import html
+from novel_downloader.core.parsers.base import BaseParser
+from novel_downloader.models import (
+    BookInfoDict,
+    ChapterDict,
+    ChapterInfoDict,
+    VolumeInfoDict,
+)
+
+# from novel_downloader.core.parsers.registry import register_parser
+
+
+# @register_parser(
+#     site_keys=["deqixs"],
+# )
+class DeqixsParser(BaseParser):
+    """
+    Parser for 得奇小说网 book pages.
+    """
+
+    ADS: set[str] = {
+        "更新不易",
+        "记得分享",
+        "(本章完)",
+    }
+
+    def parse_book_info(
+        self,
+        html_list: list[str],
+        **kwargs: Any,
+    ) -> BookInfoDict | None:
+        if not html_list:
+            return None
+
+        tree = html.fromstring(html_list[0])
+
+        # Extract book title and word count
+        book_name = tree.xpath("//div[@class='itemtxt']/h1/a/text()")[0].strip()
+        word_count = tree.xpath("//div[@class='itemtxt']/h1/i/text()")[0].strip()
+
+        # Extract serialization status and genre tags
+        spans = tree.xpath("//div[@class='itemtxt']/p[1]/span/text()")
+        serial_status = spans[0].strip() if spans else ""
+        tags = [s.strip() for s in spans[1:-1]] if len(spans) > 2 else []
+
+        # Extract author
+        author_text = tree.xpath("//div[@class='itemtxt']/p[2]/a/text()")[0]
+        author = author_text.replace("作者：", "").strip()
+
+        # Extract cover URL
+        cover_src = tree.xpath("//div[@class='item']//a/img/@src")[0]
+        cover_url = "https:" + cover_src if cover_src.startswith("//") else cover_src
+
+        # Extract last update time
+        update_raw = tree.xpath("//h2[@id='dir']/span/text()")[0].strip()
+        update_time = update_raw.replace("更新时间：", "").strip()
+
+        # Extract summary paragraphs (first description block)
+        paras = tree.xpath("(//div[@class='des bb'])[1]/p/text()")
+        summary = "\n".join(p.strip() for p in paras if p.strip())
+
+        # Extract chapters list
+        chapter_nodes = tree.xpath("//div[@id='list']//ul/li/a")
+        chapters: list[ChapterInfoDict] = []
+        for a in chapter_nodes:
+            href = a.get("href")
+            chapter_id = href.split("/")[-1].replace(".html", "")
+            title = a.text_content().strip()
+            chapters.append({"title": title, "url": href, "chapterId": chapter_id})
+
+        volumes: list[VolumeInfoDict] = [{"volume_name": "正文", "chapters": chapters}]
+
+        return {
+            "book_name": book_name,
+            "author": author,
+            "cover_url": cover_url,
+            "update_time": update_time,
+            "serial_status": serial_status,
+            "word_count": word_count,
+            "summary": summary,
+            "tags": tags,
+            "volumes": volumes,
+            "extra": {},
+        }
+
+    def parse_chapter(
+        self,
+        html_list: list[str],
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> ChapterDict | None:
+        if not html_list:
+            return None
+
+        title_text = ""
+        contents: list[str] = []
+        for curr_html in html_list:
+            tree = html.fromstring(curr_html)
+            # Extract title once
+            if not title_text:
+                full_title = tree.xpath("string(//div[@class='submenu']/h1)")
+                if ">" in full_title:
+                    title_text = full_title.split(">", 1)[1].strip()
+                else:
+                    title_text = full_title.strip()
+            # Extract paragraphs
+            for p in tree.xpath("//div[@class='con']/p"):
+                text = p.text_content().strip()
+                # Filter out ads or empty paragraphs
+                if not text or any(ad in text for ad in self.ADS):
+                    continue
+                contents.append(text)
+
+        content = "\n".join(contents)
+        if not content:
+            return None
+
+        return {
+            "id": chapter_id,
+            "title": title_text,
+            "content": content,
+            "extra": {"site": "deqixs"},
+        }

novel_downloader/core/archived/deqixs/searcher.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.deqixs.searcher
+----------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["deqixs"],
+# )
+class DeqixsSearcher(BaseSearcher):
+    site_name = "deqixs"
+    priority = 20
+    BASE_URL = "https://www.deqixs.com"
+    SEARCH_URL = "https://www.deqixs.com/tag/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {"key": keyword}
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath("//div[@class='container']/div[@class='item']")
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+
+            href = row.xpath(".//h3/a/@href")[0]
+            book_id = href.strip("/ ").split("/")[-1]
+            if not book_id:
+                continue
+            book_url = cls.BASE_URL + href
+            img_src = row.xpath(".//a/img/@src")[0]
+            cover_url = "https:" + img_src if img_src.startswith("//") else img_src
+            title = row.xpath(".//h3/a/text()")[0].strip()
+
+            author_text = row.xpath(".//p[2]/a/text()")[0]
+            author = author_text.replace("作者：", "").strip()
+
+            spans = row.xpath(".//p[1]/span/text()")
+            word_count = spans[2].strip() if len(spans) > 2 else ""
+
+            # Extract latest chapter and update date
+            first_li = row.xpath(".//ul/li")[0]
+            update_date = first_li.xpath("./i/text()")[0].strip()
+            latest_chapter = first_li.xpath("./a/text()")[0].strip()
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter=latest_chapter,
+                    update_date=update_date,
+                    word_count=word_count,
+                    priority=prio,
+                )
+            )
+        return results

novel_downloader/core/{searchers/qidian.py → archived/qidian/searcher.py}

@@ -1,14 +1,13 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.searchers.qidian
---------------------------------------
+novel_downloader.core.archived.qidian.searcher
+----------------------------------------------
 
 """
 
 import logging
 
 from lxml import html
-
 from novel_downloader.core.searchers.base import BaseSearcher
 from novel_downloader.models import SearchResult
 
@@ -28,35 +27,21 @@ class QidianSearcher(BaseSearcher):
     SEARCH_URL = "https://www.qidian.com/so/{query}.html"
 
     @classmethod
-    def _fetch_html(cls, keyword: str) -> str:
-        """
-        Fetch raw HTML from Qidian's search page.
-
-        :param keyword: The search term to query on Qidian.
-        :return: HTML text of the search results page, or an empty string on fail.
-        """
+    async def _fetch_html(cls, keyword: str) -> str:
         url = cls.SEARCH_URL.format(query=cls._quote(keyword))
         try:
-            response = cls._http_get(url)
-            return response.text
+            async with (await cls._http_get(url)) as resp:
+                return await cls._response_to_str(resp)
         except Exception:
             logger.error(
                 "Failed to fetch HTML for keyword '%s' from '%s'",
                 keyword,
                 url,
-                exc_info=True,
             )
             return ""
 
     @classmethod
     def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
-        """
-        Parse raw HTML from Qidian search results into list of SearchResult.
-
-        :param html_str: Raw HTML string from Qidian search results page.
-        :param limit: Maximum number of results to return, or None for all.
-        :return: List of SearchResult dicts.
-        """
         doc = html.fromstring(html_str)
         items = doc.xpath(
             '//div[@id="result-list"]//li[contains(@class, "res-book-item")]'
@@ -68,6 +53,8 @@ class QidianSearcher(BaseSearcher):
             if limit is not None and idx >= limit:
                 break
             book_id = item.get("data-bid")
+            if not book_id:
+                continue
             title_elem = item.xpath('.//h3[@class="book-info-title"]/a')[0]
             title = title_elem.text_content().strip()
             author_nodes = item.xpath(
@@ -79,8 +66,13 @@ class QidianSearcher(BaseSearcher):
                 SearchResult(
                     site=cls.site_name,
                     book_id=book_id,
+                    book_url="",
+                    cover_url="",
                     title=title,
                     author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
                     priority=prio,
                 )
             )

novel_downloader/core/archived/wanbengo/searcher.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.archived.wanbengo.searcher
+------------------------------------------------
+
+"""
+
+import logging
+
+from lxml import html
+from novel_downloader.core.searchers.base import BaseSearcher
+from novel_downloader.models import SearchResult
+
+# from novel_downloader.core.searchers.registry import register_searcher
+
+logger = logging.getLogger(__name__)
+
+
+# @register_searcher(
+#     site_keys=["wanbengo"],
+# )
+class WanbengoSearcher(BaseSearcher):
+    site_name = "wanbengo"
+    priority = 30
+    BASE_URL = "https://www.wanbengo.com"
+    SEARCH_URL = "https://www.sososhu.com/"
+
+    @classmethod
+    async def _fetch_html(cls, keyword: str) -> str:
+        params = {
+            "q": keyword,
+            "site": "wbsz",
+        }
+        try:
+            async with (await cls._http_get(cls.SEARCH_URL, params=params)) as resp:
+                return await cls._response_to_str(resp)
+        except Exception:
+            logger.error(
+                "Failed to fetch HTML for keyword '%s' from '%s'",
+                keyword,
+                cls.SEARCH_URL,
+            )
+            return ""
+
+    @classmethod
+    def _parse_html(cls, html_str: str, limit: int | None = None) -> list[SearchResult]:
+        doc = html.fromstring(html_str)
+        rows = doc.xpath(
+            "//div[contains(@class,'so_list')]//div[contains(@class,'hot')]//div[contains(@class,'item')]"
+        )
+        results: list[SearchResult] = []
+
+        for idx, row in enumerate(rows):
+            if limit is not None and idx >= limit:
+                break
+            a_nodes = row.xpath(".//dl/dt/a[1]")
+            a = a_nodes[0] if a_nodes else None
+            href = a.get("href") if a is not None else ""
+            if not href:
+                continue
+
+            book_url = cls._restore_url(cls._abs_url(href))
+            book_id = cls._book_id_from_url(book_url) if book_url else ""
+
+            title = (a.text_content() if a is not None else "").strip()
+            author = cls._first_str(row.xpath(".//dl/dt/span[1]/text()"))
+            cover_url = cls._first_str(
+                row.xpath(".//div[contains(@class,'image')]//img/@src")
+            )
+
+            # Compute priority
+            prio = cls.priority + idx
+
+            results.append(
+                SearchResult(
+                    site=cls.site_name,
+                    book_id=book_id,
+                    book_url=book_url,
+                    cover_url=cover_url,
+                    title=title,
+                    author=author,
+                    latest_chapter="-",
+                    update_date="-",
+                    word_count="-",
+                    priority=prio,
+                )
+            )
+        return results
+
+    @staticmethod
+    def _restore_url(url: str) -> str:
+        return url.replace("www.wbsz.org", "www.wanbengo.com")
+
+    @staticmethod
+    def _book_id_from_url(url: str) -> str:
+        tail = url.split("wanbengo.com", 1)[-1]
+        tail = tail.strip("/")
+        return tail.replace("/", "-")