novel-downloader 1.4.5__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/fetchers/aaatxt.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.aaatxt
+-------------------------------------
+
+"""
+
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["aaatxt"],
+)
+class AaatxtSession(BaseSession):
+    """
+    A session class for interacting with the 3A电子书 (www.aaatxt.com) novel website.
+    """
+
+    BOOK_INFO_URL = "http://www.aaatxt.com/shu/{book_id}.html"
+    CHAPTER_URL = "http://www.aaatxt.com/yuedu/{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("aaatxt", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(chapter_id=chapter_id)
+        return [await self.fetch(url, encoding="gb2312", **kwargs)]
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(chapter_id=chapter_id)

novel_downloader/core/fetchers/{biquge/session.py → b520.py}

@@ -1,19 +1,23 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.fetchers.biquge.session
----------------------------------------------
+novel_downloader.core.fetchers.b520
+-----------------------------------
 
 """
 
 from typing import Any
 
 from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
 from novel_downloader.models import FetcherConfig
 
 
+@register_fetcher(
+    site_keys=["biquge", "bqg", "b520"],
+)
 class BiqugeSession(BaseSession):
     """
-    A session class for interacting with the Biquge (www.b520.cc) novel website.
+    A session class for interacting with the 笔趣阁 (www.b520.cc) novel website.
     """
 
     BOOK_INFO_URL = "http://www.b520.cc/{book_id}/"
@@ -36,7 +40,7 @@ class BiqugeSession(BaseSession):
         Fetch the raw HTML of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         url = self.book_info_url(book_id=book_id)
         return [await self.fetch(url, **kwargs)]
@@ -52,10 +56,10 @@ class BiqugeSession(BaseSession):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
+        :return: The page content as string list.
         """
         url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
-        return [await self.fetch(url, **kwargs)]
+        return [await self.fetch(url, encoding="gbk", **kwargs)]
 
     @classmethod
     def book_info_url(cls, book_id: str) -> str:
@@ -77,7 +81,3 @@ class BiqugeSession(BaseSession):
         :return: Fully qualified chapter URL.
         """
         return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)
-
-    @property
-    def hostname(self) -> str:
-        return "www.b520.cc"

novel_downloader/core/fetchers/{base/session.py → base.py}

@@ -1,19 +1,16 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.fetchers.base.session
--------------------------------------------
+novel_downloader.core.fetchers.base
+-----------------------------------
 
-This module defines the BaseSession class, which provides asynchronous
-HTTP request capabilities using aiohttp. It maintains a persistent
-client session and supports retries, headers, timeout configurations,
-cookie handling, and defines abstract methods for subclasses.
+Abstract base class providing common HTTP session handling for fetchers.
 """
 
-
 import abc
 import json
 import logging
 import types
+from collections.abc import Mapping
 from typing import Any, Self
 
 import aiohttp
@@ -21,12 +18,13 @@ from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
 
 from novel_downloader.core.interfaces import FetcherProtocol
 from novel_downloader.models import FetcherConfig, LoginField
+from novel_downloader.utils import (
+    async_jitter_sleep,
+)
 from novel_downloader.utils.constants import (
     DATA_DIR,
     DEFAULT_USER_HEADERS,
 )
-from novel_downloader.utils.cookies import parse_cookie_expires
-from novel_downloader.utils.time_utils import async_sleep_with_random_delay
 
 from .rate_limiter import TokenBucketRateLimiter
 
@@ -65,7 +63,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
         self._session: ClientSession | None = None
         self._rate_limiter: TokenBucketRateLimiter | None = None
 
-        if config.max_rps is not None and config.max_rps > 0:
+        if config.max_rps > 0:
             self._rate_limiter = TokenBucketRateLimiter(config.max_rps)
 
         self.logger = logging.getLogger(f"{self.__class__.__name__}")
@@ -80,6 +78,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
     ) -> bool:
         """
         Attempt to log in asynchronously.
+
         :returns: True if login succeeded.
         """
         return False
@@ -94,7 +93,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
         Fetch the raw HTML (or JSON) of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         ...
 
@@ -110,7 +109,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
+        :return: The page content as string list.
         """
         ...
 
@@ -156,7 +155,12 @@ class BaseSession(FetcherProtocol, abc.ABC):
             await self._session.close()
         self._session = None
 
-    async def fetch(self, url: str, **kwargs: Any) -> str:
+    async def fetch(
+        self,
+        url: str,
+        encoding: str | None = None,
+        **kwargs: Any,
+    ) -> str:
         """
         Fetch the content from the given URL asynchronously, with retry support.
 
@@ -172,11 +176,10 @@ class BaseSession(FetcherProtocol, abc.ABC):
             try:
                 async with self.session.get(url, **kwargs) as resp:
                     resp.raise_for_status()
-                    text: str = await resp.text()
-                    return text
+                    return await self._response_to_str(resp, encoding)
             except aiohttp.ClientError:
                 if attempt < self.retry_times:
-                    await async_sleep_with_random_delay(
+                    await async_jitter_sleep(
                         self.backoff_factor,
                         mul_spread=1.1,
                         max_sleep=self.backoff_factor + 2,
@@ -241,8 +244,12 @@ class BaseSession(FetcherProtocol, abc.ABC):
             return False
         try:
             storage = json.loads(self._state_file.read_text(encoding="utf-8"))
-            for c in storage.get("cookies", []):
-                self._session.cookie_jar.update_cookies({c["name"]: c["value"]})
+            raw_cookies = storage.get("cookies", [])
+            cookie_dict = self._filter_cookies(raw_cookies)
+
+            if cookie_dict:
+                self._session.cookie_jar.update_cookies(cookie_dict)
+
             self._is_logged_in = await self._check_login_status()
             return self._is_logged_in
         except Exception as e:
@@ -272,12 +279,6 @@ class BaseSession(FetcherProtocol, abc.ABC):
                     {
                         "name": cookie.key,
                         "value": cookie.value,
-                        "domain": cookie.get("domain", ""),
-                        "path": cookie.get("path", "/"),
-                        "expires": parse_cookie_expires(cookie.get("expires")),
-                        "httpOnly": bool(cookie.get("httponly", False)),
-                        "secure": bool(cookie.get("secure", False)),
-                        "sameSite": cookie.get("samesite") or "Lax",
                     }
                 )
             storage_state = {
@@ -294,21 +295,6 @@ class BaseSession(FetcherProtocol, abc.ABC):
             self.logger.warning("Failed to save state: %s", e)
             return False
 
-    async def set_interactive_mode(self, enable: bool) -> bool:
-        """
-        Enable or disable interactive mode for manual login.
-
-        :param enable: True to enable, False to disable interactive mode.
-        :return: True if operation or login check succeeded, False otherwise.
-        """
-        return False
-
-    def get_cookie_value(self, key: str) -> str | None:
-        for cookie in self.session.cookie_jar:
-            if cookie.key == key:
-                return str(cookie.value)
-        return None
-
     def update_cookies(
         self,
         cookies: dict[str, str],
@@ -340,18 +326,10 @@ class BaseSession(FetcherProtocol, abc.ABC):
         """
         return False
 
-    @property
-    def hostname(self) -> str:
-        return ""
-
     @property
     def site(self) -> str:
         return self._site
 
-    @property
-    def requester_type(self) -> str:
-        return "session"
-
     @property
     def is_logged_in(self) -> bool:
         """
@@ -405,6 +383,44 @@ class BaseSession(FetcherProtocol, abc.ABC):
             return dict(self._session.headers)
         return self._headers.copy()
 
+    @staticmethod
+    def _filter_cookies(
+        raw_cookies: list[Mapping[str, Any]],
+    ) -> dict[str, str]:
+        """
+        Hook:
+        take the raw list of cookie-dicts loaded from storage_state
+        and return a simple name -> value mapping.
+        """
+        return {c["name"]: c["value"] for c in raw_cookies}
+
+    @staticmethod
+    async def _response_to_str(
+        resp: ClientResponse,
+        encoding: str | None = None,
+    ) -> str:
+        """
+        Read the full body of resp as text. First try the declared charset,
+        then on UnicodeDecodeError fall back to a lenient utf-8 decode.
+        """
+        data: bytes = await resp.read()
+        encodings = [
+            encoding,
+            resp.charset,
+            "gb2312",
+            "gb18030",
+            "gbk",
+            "utf-8",
+        ]
+        encodings_list: list[str] = [e for e in encodings if e]
+        for enc in encodings_list:
+            try:
+                return data.decode(enc)
+            except UnicodeDecodeError:
+                continue
+        encoding = encoding or "utf-8"
+        return data.decode(encoding, errors="ignore")
+
     async def __aenter__(self) -> Self:
         if self._session is None or self._session.closed:
             await self.init()

novel_downloader/core/fetchers/biquyuedu.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.biquyuedu
+----------------------------------------
+
+"""
+
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["biquyuedu"],
+)
+class BiquyueduSession(BaseSession):
+    """
+    A session class for interacting with the 精彩小说 (biquyuedu.com) novel website.
+    """
+
+    BOOK_INFO_URL = "https://biquyuedu.com/novel/{book_id}.html"
+    CHAPTER_URL = "https://biquyuedu.com/novel/{book_id}/{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("biquyuedu", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
+        return [await self.fetch(url, **kwargs)]
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)

novel_downloader/core/fetchers/dxmwx.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.dxmwx
+------------------------------------
+
+"""
+
+import asyncio
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["dxmwx"],
+)
+class DxmwxSession(BaseSession):
+    """
+    A session class for interacting with the 大熊猫文学网 (www.dxmwx.org) novel website.
+    """
+
+    BOOK_INFO_URL = "https://{base_url}/book/{book_id}.html"
+    BOOK_CATALOG_URL = "https://{base_url}/chapter/{book_id}.html"
+    CHAPTER_URL = "https://{base_url}/read/{book_id}_{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("dxmwx", config, cookies, **kwargs)
+        self.base_url = (
+            "www.dxmwx.org" if config.locale_style == "simplified" else "tw.dxmwx.org"
+        )
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        Order: [info, catalog]
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        info_url = self.book_info_url(base_url=self.base_url, book_id=book_id)
+        catalog_url = self.book_catalog_url(base_url=self.base_url, book_id=book_id)
+
+        info_html, catalog_html = await asyncio.gather(
+            self.fetch(info_url, **kwargs),
+            self.fetch(catalog_url, **kwargs),
+        )
+        return [info_html, catalog_html]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(
+            base_url=self.base_url, book_id=book_id, chapter_id=chapter_id
+        )
+        return [await self.fetch(url, **kwargs)]
+
+    @classmethod
+    def book_info_url(cls, base_url: str, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(base_url=base_url, book_id=book_id)
+
+    @classmethod
+    def book_catalog_url(cls, base_url: str, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's catalog page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified catalog page URL.
+        """
+        return cls.BOOK_CATALOG_URL.format(base_url=base_url, book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, base_url: str, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(
+            base_url=base_url, book_id=book_id, chapter_id=chapter_id
+        )

novel_downloader/core/fetchers/eightnovel.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.eightnovel
+-----------------------------------------
+
+"""
+
+import re
+from re import Pattern
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["8novel", "eightnovel"],
+)
+class EightnovelSession(BaseSession):
+    """
+    A session class for interacting with the 无限轻小说 (www.8novel.com) novel website.
+    """
+
+    BOOK_INFO_URL = "https://www.8novel.com/novelbooks/{book_id}/"
+    CHAPTER_URL = "https://article.8novel.com/read/{book_id}/?{chapter_id}"
+    CHAPTER_CONTENT_URL = (
+        "https://article.8novel.com/txt/1/{book_id}/{chapter_id}{seed_segment}.html"
+    )
+
+    _SPLIT_STR_PATTERN = re.compile(
+        r'["\']([^"\']+)["\']\s*\.split\s*\(\s*["\']\s*,\s*["\']\s*\)', re.DOTALL
+    )
+    _DIGIT_LIST_PATTERN: Pattern[str] = re.compile(r"^\d+(?:,\d+)*$")
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("eightnovel", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        Order: [chap_info, content]
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
+        chapter_html = await self.fetch(url, **kwargs)
+        url_seed = self._extract_url_seed(chapter_html)
+        content_url = self._build_chapter_content_url(
+            seed=url_seed,
+            book_id=book_id,
+            chapter_id=chapter_id,
+        )
+        content_html = await self.fetch(content_url, **kwargs)
+
+        return [chapter_html, content_html]
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)
+
+    @classmethod
+    def _extract_url_seed(cls, html_str: str) -> str:
+        """
+        From the given HTML/JS source, find all string literals
+        of the form "...".split(","), pick the ones that may contain seed,
+        and return the last value.
+        """
+        split_literals: list[str] = cls._SPLIT_STR_PATTERN.findall(html_str)
+
+        numeric_lists = [
+            lit for lit in split_literals if cls._DIGIT_LIST_PATTERN.fullmatch(lit)
+        ]
+
+        if not numeric_lists:
+            return ""
+
+        last_list = numeric_lists[-1]
+        return last_list.split(",")[-1]
+
+    @classmethod
+    def _build_chapter_content_url(
+        cls, seed: str, book_id: str, chapter_id: str
+    ) -> str:
+        """
+        Slices out a 5-character segment of `seed` at offset
+        and build content url.
+        """
+        # Compute start index and slice out 5 chars
+        start = (int(chapter_id) * 3) % 100
+        seed_segment = seed[start : start + 5]
+
+        return cls.CHAPTER_CONTENT_URL.format(
+            book_id=book_id, chapter_id=chapter_id, seed_segment=seed_segment
+        )