novel-downloader 1.5.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

novel_downloader/core/fetchers/{base/session.py → base.py}

@@ -1,19 +1,16 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.fetchers.base.session
--------------------------------------------
+novel_downloader.core.fetchers.base
+-----------------------------------
 
-This module defines the BaseSession class, which provides asynchronous
-HTTP request capabilities using aiohttp. It maintains a persistent
-client session and supports retries, headers, timeout configurations,
-cookie handling, and defines abstract methods for subclasses.
+Abstract base class providing common HTTP session handling for fetchers.
 """
 
-
 import abc
 import json
 import logging
 import types
+from collections.abc import Mapping
 from typing import Any, Self
 
 import aiohttp
@@ -22,8 +19,7 @@ from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
 from novel_downloader.core.interfaces import FetcherProtocol
 from novel_downloader.models import FetcherConfig, LoginField
 from novel_downloader.utils import (
-    async_sleep_with_random_delay,
-    parse_cookie_expires,
+    async_jitter_sleep,
 )
 from novel_downloader.utils.constants import (
     DATA_DIR,
@@ -67,7 +63,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
         self._session: ClientSession | None = None
         self._rate_limiter: TokenBucketRateLimiter | None = None
 
-        if config.max_rps is not None and config.max_rps > 0:
+        if config.max_rps > 0:
             self._rate_limiter = TokenBucketRateLimiter(config.max_rps)
 
         self.logger = logging.getLogger(f"{self.__class__.__name__}")
@@ -82,6 +78,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
     ) -> bool:
         """
         Attempt to log in asynchronously.
+
         :returns: True if login succeeded.
         """
         return False
@@ -96,7 +93,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
         Fetch the raw HTML (or JSON) of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         ...
 
@@ -112,7 +109,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
+        :return: The page content as string list.
         """
         ...
 
@@ -182,7 +179,7 @@ class BaseSession(FetcherProtocol, abc.ABC):
                     return await self._response_to_str(resp, encoding)
             except aiohttp.ClientError:
                 if attempt < self.retry_times:
-                    await async_sleep_with_random_delay(
+                    await async_jitter_sleep(
                         self.backoff_factor,
                         mul_spread=1.1,
                         max_sleep=self.backoff_factor + 2,
@@ -247,8 +244,12 @@ class BaseSession(FetcherProtocol, abc.ABC):
             return False
         try:
             storage = json.loads(self._state_file.read_text(encoding="utf-8"))
-            for c in storage.get("cookies", []):
-                self._session.cookie_jar.update_cookies({c["name"]: c["value"]})
+            raw_cookies = storage.get("cookies", [])
+            cookie_dict = self._filter_cookies(raw_cookies)
+
+            if cookie_dict:
+                self._session.cookie_jar.update_cookies(cookie_dict)
+
             self._is_logged_in = await self._check_login_status()
             return self._is_logged_in
         except Exception as e:
@@ -278,12 +279,6 @@ class BaseSession(FetcherProtocol, abc.ABC):
                     {
                         "name": cookie.key,
                         "value": cookie.value,
-                        "domain": cookie.get("domain", ""),
-                        "path": cookie.get("path", "/"),
-                        "expires": parse_cookie_expires(cookie.get("expires")),
-                        "httpOnly": bool(cookie.get("httponly", False)),
-                        "secure": bool(cookie.get("secure", False)),
-                        "sameSite": cookie.get("samesite") or "Lax",
                     }
                 )
             storage_state = {
@@ -300,21 +295,6 @@ class BaseSession(FetcherProtocol, abc.ABC):
             self.logger.warning("Failed to save state: %s", e)
             return False
 
-    async def set_interactive_mode(self, enable: bool) -> bool:
-        """
-        Enable or disable interactive mode for manual login.
-
-        :param enable: True to enable, False to disable interactive mode.
-        :return: True if operation or login check succeeded, False otherwise.
-        """
-        return False
-
-    def get_cookie_value(self, key: str) -> str | None:
-        for cookie in self.session.cookie_jar:
-            if cookie.key == key:
-                return str(cookie.value)
-        return None
-
     def update_cookies(
         self,
         cookies: dict[str, str],
@@ -346,18 +326,10 @@ class BaseSession(FetcherProtocol, abc.ABC):
         """
         return False
 
-    @property
-    def hostname(self) -> str:
-        return ""
-
     @property
     def site(self) -> str:
         return self._site
 
-    @property
-    def requester_type(self) -> str:
-        return "session"
-
     @property
     def is_logged_in(self) -> bool:
         """
@@ -411,6 +383,17 @@ class BaseSession(FetcherProtocol, abc.ABC):
             return dict(self._session.headers)
         return self._headers.copy()
 
+    @staticmethod
+    def _filter_cookies(
+        raw_cookies: list[Mapping[str, Any]],
+    ) -> dict[str, str]:
+        """
+        Hook:
+        take the raw list of cookie-dicts loaded from storage_state
+        and return a simple name -> value mapping.
+        """
+        return {c["name"]: c["value"] for c in raw_cookies}
+
     @staticmethod
     async def _response_to_str(
         resp: ClientResponse,
@@ -421,14 +404,22 @@ class BaseSession(FetcherProtocol, abc.ABC):
         then on UnicodeDecodeError fall back to a lenient utf-8 decode.
         """
         data: bytes = await resp.read()
-        encodings = [encoding, resp.charset, "utf-8", "gb18030", "gbk"]
+        encodings = [
+            encoding,
+            resp.charset,
+            "gb2312",
+            "gb18030",
+            "gbk",
+            "utf-8",
+        ]
         encodings_list: list[str] = [e for e in encodings if e]
         for enc in encodings_list:
             try:
                 return data.decode(enc)
             except UnicodeDecodeError:
                 continue
-        return data.decode("utf-8", errors="ignore")
+        encoding = encoding or "utf-8"
+        return data.decode(encoding, errors="ignore")
 
     async def __aenter__(self) -> Self:
         if self._session is None or self._session.closed:

novel_downloader/core/fetchers/{biquge/browser.py → biquyuedu.py}

@@ -1,36 +1,35 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.fetchers.biquge.browser
----------------------------------------------
+novel_downloader.core.fetchers.biquyuedu
+----------------------------------------
 
 """
 
 from typing import Any
 
-from novel_downloader.core.fetchers.base import BaseBrowser
+from novel_downloader.core.fetchers.base import BaseSession
 from novel_downloader.core.fetchers.registry import register_fetcher
 from novel_downloader.models import FetcherConfig
 
 
 @register_fetcher(
-    site_keys=["biquge", "bqg"],
-    backends=["browser"],
+    site_keys=["biquyuedu"],
 )
-class BiqugeBrowser(BaseBrowser):
+class BiquyueduSession(BaseSession):
     """
-    A browser class for interacting with the Biquge (www.b520.cc) novel website.
+    A session class for interacting with the 精彩小说 (biquyuedu.com) novel website.
     """
 
-    BOOK_INFO_URL = "http://www.b520.cc/{book_id}/"
-    CHAPTER_URL = "http://www.b520.cc/{book_id}/{chapter_id}.html"
+    BOOK_INFO_URL = "https://biquyuedu.com/novel/{book_id}.html"
+    CHAPTER_URL = "https://biquyuedu.com/novel/{book_id}/{chapter_id}.html"
 
     def __init__(
         self,
         config: FetcherConfig,
-        reuse_page: bool = False,
+        cookies: dict[str, str] | None = None,
         **kwargs: Any,
     ) -> None:
-        super().__init__("biquge", config, reuse_page, **kwargs)
+        super().__init__("biquyuedu", config, cookies, **kwargs)
 
     async def get_book_info(
         self,
@@ -41,7 +40,7 @@ class BiqugeBrowser(BaseBrowser):
         Fetch the raw HTML of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         url = self.book_info_url(book_id=book_id)
         return [await self.fetch(url, **kwargs)]
@@ -57,7 +56,7 @@ class BiqugeBrowser(BaseBrowser):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
+        :return: The page content as string list.
         """
         url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
         return [await self.fetch(url, **kwargs)]
@@ -82,7 +81,3 @@ class BiqugeBrowser(BaseBrowser):
         :return: Fully qualified chapter URL.
         """
         return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)
-
-    @property
-    def hostname(self) -> str:
-        return "www.b520.cc"

novel_downloader/core/fetchers/dxmwx.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.dxmwx
+------------------------------------
+
+"""
+
+import asyncio
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["dxmwx"],
+)
+class DxmwxSession(BaseSession):
+    """
+    A session class for interacting with the 大熊猫文学网 (www.dxmwx.org) novel website.
+    """
+
+    BOOK_INFO_URL = "https://{base_url}/book/{book_id}.html"
+    BOOK_CATALOG_URL = "https://{base_url}/chapter/{book_id}.html"
+    CHAPTER_URL = "https://{base_url}/read/{book_id}_{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("dxmwx", config, cookies, **kwargs)
+        self.base_url = (
+            "www.dxmwx.org" if config.locale_style == "simplified" else "tw.dxmwx.org"
+        )
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        Order: [info, catalog]
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        info_url = self.book_info_url(base_url=self.base_url, book_id=book_id)
+        catalog_url = self.book_catalog_url(base_url=self.base_url, book_id=book_id)
+
+        info_html, catalog_html = await asyncio.gather(
+            self.fetch(info_url, **kwargs),
+            self.fetch(catalog_url, **kwargs),
+        )
+        return [info_html, catalog_html]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(
+            base_url=self.base_url, book_id=book_id, chapter_id=chapter_id
+        )
+        return [await self.fetch(url, **kwargs)]
+
+    @classmethod
+    def book_info_url(cls, base_url: str, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(base_url=base_url, book_id=book_id)
+
+    @classmethod
+    def book_catalog_url(cls, base_url: str, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's catalog page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified catalog page URL.
+        """
+        return cls.BOOK_CATALOG_URL.format(base_url=base_url, book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, base_url: str, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(
+            base_url=base_url, book_id=book_id, chapter_id=chapter_id
+        )

novel_downloader/core/fetchers/eightnovel.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.eightnovel
+-----------------------------------------
+
+"""
+
+import re
+from re import Pattern
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["8novel", "eightnovel"],
+)
+class EightnovelSession(BaseSession):
+    """
+    A session class for interacting with the 无限轻小说 (www.8novel.com) novel website.
+    """
+
+    BOOK_INFO_URL = "https://www.8novel.com/novelbooks/{book_id}/"
+    CHAPTER_URL = "https://article.8novel.com/read/{book_id}/?{chapter_id}"
+    CHAPTER_CONTENT_URL = (
+        "https://article.8novel.com/txt/1/{book_id}/{chapter_id}{seed_segment}.html"
+    )
+
+    _SPLIT_STR_PATTERN = re.compile(
+        r'["\']([^"\']+)["\']\s*\.split\s*\(\s*["\']\s*,\s*["\']\s*\)', re.DOTALL
+    )
+    _DIGIT_LIST_PATTERN: Pattern[str] = re.compile(r"^\d+(?:,\d+)*$")
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("eightnovel", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        Order: [chap_info, content]
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
+        chapter_html = await self.fetch(url, **kwargs)
+        url_seed = self._extract_url_seed(chapter_html)
+        content_url = self._build_chapter_content_url(
+            seed=url_seed,
+            book_id=book_id,
+            chapter_id=chapter_id,
+        )
+        content_html = await self.fetch(content_url, **kwargs)
+
+        return [chapter_html, content_html]
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)
+
+    @classmethod
+    def _extract_url_seed(cls, html_str: str) -> str:
+        """
+        From the given HTML/JS source, find all string literals
+        of the form "...".split(","), pick the ones that may contain seed,
+        and return the last value.
+        """
+        split_literals: list[str] = cls._SPLIT_STR_PATTERN.findall(html_str)
+
+        numeric_lists = [
+            lit for lit in split_literals if cls._DIGIT_LIST_PATTERN.fullmatch(lit)
+        ]
+
+        if not numeric_lists:
+            return ""
+
+        last_list = numeric_lists[-1]
+        return last_list.split(",")[-1]
+
+    @classmethod
+    def _build_chapter_content_url(
+        cls, seed: str, book_id: str, chapter_id: str
+    ) -> str:
+        """
+        Slices out a 5-character segment of `seed` at offset
+        and build content url.
+        """
+        # Compute start index and slice out 5 chars
+        start = (int(chapter_id) * 3) % 100
+        seed_segment = seed[start : start + 5]
+
+        return cls.CHAPTER_CONTENT_URL.format(
+            book_id=book_id, chapter_id=chapter_id, seed_segment=seed_segment
+        )

novel_downloader/core/fetchers/{esjzone/session.py → esjzone.py}

@@ -1,26 +1,26 @@
 #!/usr/bin/env python3
 """
-novel_downloader.core.fetchers.esjzone.session
-----------------------------------------------
+novel_downloader.core.fetchers.esjzone
+--------------------------------------
 
 """
 
 import re
+from collections.abc import Mapping
 from typing import Any
 
 from novel_downloader.core.fetchers.base import BaseSession
 from novel_downloader.core.fetchers.registry import register_fetcher
 from novel_downloader.models import FetcherConfig, LoginField
-from novel_downloader.utils import async_sleep_with_random_delay
+from novel_downloader.utils import async_jitter_sleep
 
 
 @register_fetcher(
     site_keys=["esjzone"],
-    backends=["session"],
 )
 class EsjzoneSession(BaseSession):
     """
-    A session class for interacting with the esjzone (www.esjzone.cc) novel website.
+    A session class for interacting with the ESJ Zone (www.esjzone.cc) novel website.
     """
 
     BOOKCASE_URL = "https://www.esjzone.cc/my/favorite"
@@ -68,7 +68,7 @@ class EsjzoneSession(BaseSession):
             ):
                 self._is_logged_in = True
                 return True
-            await async_sleep_with_random_delay(
+            await async_jitter_sleep(
                 self.backoff_factor,
                 mul_spread=1.1,
                 max_sleep=self.backoff_factor + 2,
@@ -86,7 +86,7 @@ class EsjzoneSession(BaseSession):
         Fetch the raw HTML of the book info page asynchronously.
 
         :param book_id: The book identifier.
-        :return: The page content as a string.
+        :return: The page content as string list.
         """
         url = self.book_info_url(book_id=book_id)
         return [await self.fetch(url, **kwargs)]
@@ -102,7 +102,7 @@ class EsjzoneSession(BaseSession):
 
         :param book_id: The book identifier.
         :param chapter_id: The chapter identifier.
-        :return: The chapter content as a string.
+        :return: The page content as string list.
         """
         url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
         return [await self.fetch(url, **kwargs)]
@@ -170,10 +170,6 @@ class EsjzoneSession(BaseSession):
         """
         return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)
 
-    @property
-    def hostname(self) -> str:
-        return "www.esjzone.cc"
-
     async def _api_login(self, username: str, password: str) -> bool:
         """
         Login to the API using a 2-step token-based process.
@@ -234,3 +230,14 @@ class EsjzoneSession(BaseSession):
     def _extract_token(self, text: str) -> str:
         match = re.search(r"<JinJing>(.+?)</JinJing>", text)
         return match.group(1) if match else ""
+
+    @staticmethod
+    def _filter_cookies(
+        raw_cookies: list[Mapping[str, Any]],
+    ) -> dict[str, str]:
+        ALLOWED_DOMAINS = {".www.esjzone.cc", "www.esjzone.cc", ".esjzone.cc", ""}
+        return {
+            c["name"]: c["value"]
+            for c in raw_cookies
+            if c.get("domain", "") in ALLOWED_DOMAINS
+        }

novel_downloader/core/fetchers/guidaye.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.fetchers.guidaye
+--------------------------------------
+
+"""
+
+from typing import Any
+
+from novel_downloader.core.fetchers.base import BaseSession
+from novel_downloader.core.fetchers.registry import register_fetcher
+from novel_downloader.models import FetcherConfig
+
+
+@register_fetcher(
+    site_keys=["guidaye"],
+)
+class GuidayeSession(BaseSession):
+    """
+    A session class for interacting with the 名著阅读 (b.guidaye.com) novel website.
+    """
+
+    BOOK_INFO_URL = "https://b.guidaye.com/{book_id}/"
+    CHAPTER_URL = "https://b.guidaye.com/{book_id}/{chapter_id}.html"
+
+    def __init__(
+        self,
+        config: FetcherConfig,
+        cookies: dict[str, str] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__("guidaye", config, cookies, **kwargs)
+
+    async def get_book_info(
+        self,
+        book_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        :param book_id: The book identifier.
+        :return: The page content as string list.
+        """
+        book_id = book_id.replace("-", "/")
+        url = self.book_info_url(book_id=book_id)
+        return [await self.fetch(url, **kwargs)]
+
+    async def get_book_chapter(
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
+    ) -> list[str]:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :return: The page content as string list.
+        """
+        book_id = book_id.replace("-", "/")
+        url = self.chapter_url(book_id=book_id, chapter_id=chapter_id)
+        return [await self.fetch(url, **kwargs)]
+
+    @classmethod
+    def book_info_url(cls, book_id: str) -> str:
+        """
+        Construct the URL for fetching a book's info page.
+
+        :param book_id: The identifier of the book.
+        :return: Fully qualified URL for the book info page.
+        """
+        return cls.BOOK_INFO_URL.format(book_id=book_id)
+
+    @classmethod
+    def chapter_url(cls, book_id: str, chapter_id: str) -> str:
+        """
+        Construct the URL for fetching a specific chapter.
+
+        :param book_id: The identifier of the book.
+        :param chapter_id: The identifier of the chapter.
+        :return: Fully qualified chapter URL.
+        """
+        return cls.CHAPTER_URL.format(book_id=book_id, chapter_id=chapter_id)