novel-downloader 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

novel_downloader/core/parsers/qidian/shared/book_info_parser.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.parsers.qidian.shared.book_info_parser
+------------------------------------------------------------
+
+This module provides parsing of Qidian book info pages.
+
+It extracts metadata such as title, author, cover URL, update
+time, status, word count, summary, and volume-chapter structure.
+"""
+
+import logging
+import re
+from typing import Any
+
+from bs4.element import Tag
+
+from .helpers import html_to_soup
+
+logger = logging.getLogger(__name__)
+
+
+def _chapter_url_to_id(url: str) -> str:
+    """
+    Extract chapterId as the last non-empty segment of the URL.
+    """
+    return url.rstrip("/").split("/")[-1]
+
+
+def _get_volume_name(vol_div: Tag) -> str:
+    """
+    Extracts the volume title from a <div class="volume"> element
+    """
+    h3 = vol_div.select_one("h3")
+    if not h3:
+        return ""
+    for a in h3.find_all("a"):
+        a.decompose()
+    text: str = h3.get_text(strip=True)
+    return text.split(chr(183))[0].strip()
+
+
+def safe_select_text(
+    soup: Tag,
+    selector: str,
+    *,
+    separator: str = "",
+    strip: bool = False,
+    default: str = "",
+) -> str:
+    """
+    Safely select the first element matching a CSS selector and return its text.
+
+    :param soup: A BeautifulSoup Tag or sub-tree to query.
+    :param selector: A CSS selector string.
+    :param separator: Separator to use between strings when joining.
+    :param strip: Whether to strip whitespace from the result.
+    :param default: Value to return if no element is found.
+    :return: The element's text, or `default` if not found.
+    """
+    tag = soup.select_one(selector)
+    return (
+        tag.get_text(separator=separator, strip=strip)
+        if isinstance(tag, Tag)
+        else default
+    )
+
+
+def safe_select_attr(
+    soup: Tag,
+    selector: str,
+    attr: str,
+    *,
+    default: str = "",
+) -> str:
+    """
+    Safely select the first element matching a CSS selector and return one attributes.
+
+    :param soup: A BeautifulSoup Tag or sub-tree to query.
+    :param selector: A CSS selector string.
+    :param attr: The attribute name to retrieve from the selected element.
+    :param default: Value to return if no element or attribute is found.
+    :return: The attribute's value stripped of whitespace, or `default` if not found.
+    """
+    tag = soup.select_one(selector)
+    if isinstance(tag, Tag) and attr in tag.attrs:
+        value = tag.attrs[attr]
+        if isinstance(value, list):
+            return " ".join(value).strip()
+        elif isinstance(value, str):
+            return value.strip()
+    return default
+
+
+def parse_book_info(html_str: str) -> dict[str, Any]:
+    """
+    Extract metadata: title, author, cover_url, update_time, status,
+    word_count, summary, and volumes with chapters.
+
+    :param html_str: Raw HTML of the book info page.
+    :return: A dict containing book metadata.
+    """
+    info: dict[str, Any] = {}
+    try:
+        soup = html_to_soup(html_str)
+        info["book_name"] = safe_select_text(soup, "em#bookName", strip=True)
+        info["author"] = safe_select_text(soup, "a.writer", strip=True)
+        info["cover_url"] = safe_select_attr(soup, "div.book-img img", "src")
+        info["update_time"] = (
+            safe_select_text(soup, "span.book-update-time", strip=True)
+            .replace("更新时间", "")
+            .strip()
+        )
+        info["serial_status"] = safe_select_text(soup, "span.blue", strip=True)
+
+        # Word count via regex fallback
+        match = re.search(r"<em>([\d.]+)</em>\s*<cite>(.*?)字</cite>", html_str)
+        info["word_count"] = (
+            f"{match.group(1)}{match.group(2)}字" if match else "Unknown"
+        )
+
+        info["summary"] = safe_select_text(
+            soup, "div.book-intro p", separator="\n", strip=True
+        )
+        # volumes
+        vols = []
+        for vol_div in soup.select("div.volume-wrap div.volume"):
+            name = _get_volume_name(vol_div)
+            chaps = []
+            for li in vol_div.select("li"):
+                a = li.select_one("a")
+                if not isinstance(a, Tag) or "href" not in a.attrs:
+                    continue
+                href_val = a["href"]
+                if isinstance(href_val, list):
+                    href = href_val[0].strip()
+                else:
+                    href = str(href_val).strip()
+                chaps.append(
+                    {
+                        "title": a.get_text(strip=True),
+                        "url": href,
+                        "chapterId": _chapter_url_to_id(href),
+                    }
+                )
+            vols.append({"volume_name": name, "chapters": chaps})
+        info["volumes"] = vols
+    except Exception as e:
+        logger.warning("[Parser] Error parsing book info: %s", e)
+    return info

novel_downloader/core/parsers/{qidian_parser → qidian}/shared/helpers.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.parsers.qidian_parser.shared.helpers
-----------------------------------------------------------
+novel_downloader.core.parsers.qidian.shared.helpers
+---------------------------------------------------
 
 Shared utility functions for parsing Qidian browser-rendered pages.
 
@@ -14,9 +13,9 @@ This module provides reusable helpers to:
 
 import json
 import logging
-from typing import Any, Dict, Union
+from typing import Any
 
-from bs4 import BeautifulSoup
+from bs4 import BeautifulSoup, Tag
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +75,7 @@ def can_view_chapter(soup: BeautifulSoup) -> bool:
     return not (vip_status == 1 and is_buy == 0)
 
 
-def is_encrypted(content: Union[str, BeautifulSoup]) -> bool:
+def is_encrypted(content: str | BeautifulSoup) -> bool:
     """
     Return True if content is encrypted.
 
@@ -97,21 +96,21 @@ def is_encrypted(content: Union[str, BeautifulSoup]) -> bool:
     return int(chapter_info.get("cES", 0)) == 2
 
 
-def find_ssr_page_context(soup: BeautifulSoup) -> Dict[str, Any]:
+def find_ssr_page_context(soup: BeautifulSoup) -> dict[str, Any]:
     """
     Extract SSR JSON from <script id="vite-plugin-ssr_pageContext">.
     """
     try:
         tag = soup.find("script", id="vite-plugin-ssr_pageContext")
-        if tag and tag.string:
-            data: Dict[str, Any] = json.loads(tag.string.strip())
+        if isinstance(tag, Tag) and tag.string:
+            data: dict[str, Any] = json.loads(tag.string.strip())
             return data
     except Exception as e:
         logger.warning("[Parser] SSR JSON parse error: %s", e)
     return {}
 
 
-def extract_chapter_info(ssr_data: Dict[str, Any]) -> Dict[str, Any]:
+def extract_chapter_info(ssr_data: dict[str, Any]) -> dict[str, Any]:
     """
     Extract the 'chapterInfo' dictionary from the SSR page context.
 

novel_downloader/core/requesters/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 novel_downloader.core.requesters
 --------------------------------
@@ -10,20 +9,25 @@ to perform network interactions, such as logging in, sending requests,
 or interacting with browser/session-based sources.
 
 Subpackages:
-- common_requester: Handles all common-site requesting logic.
-- qidian_requester: Handles all Qidian-related requesting logic.
+- common
+- biquge
+- qidian
 """
 
-from .common_requester import (
+from .biquge import (
+    BiqugeSession,
+)
+from .common import (
     CommonAsyncSession,
     CommonSession,
 )
-from .qidian_requester import (
+from .qidian import (
     QidianBrowser,
     QidianSession,
 )
 
 __all__ = [
+    "BiqugeSession",
     "CommonAsyncSession",
     "CommonSession",
     "QidianBrowser",

novel_downloader/core/requesters/base/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+"""
+novel_downloader.core.requesters.base
+-------------------------------------
+
+"""
+
+from .async_session import BaseAsyncSession
+from .browser import BaseBrowser
+from .session import BaseSession
+
+__all__ = [
+    "BaseAsyncSession",
+    "BaseBrowser",
+    "BaseSession",
+]

novel_downloader/core/requesters/{base_async_session.py → base/async_session.py}

@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
-novel_downloader.core.requesters.base_async_session
+novel_downloader.core.requesters.base.async_session
 ---------------------------------------------------
 
 This module defines the BaseAsyncSession class, which provides asynchronous
@@ -12,8 +11,11 @@ cookie handling, and defines abstract methods for subclasses.
 
 import abc
 import asyncio
+import logging
+import random
 import time
-from typing import Any, Dict, Optional, Union
+import types
+from typing import Any, Literal, Self
 
 import aiohttp
 from aiohttp import ClientResponse, ClientSession, ClientTimeout, TCPConnector
@@ -25,7 +27,8 @@ from novel_downloader.utils.constants import DEFAULT_USER_HEADERS
 
 class RateLimiter:
     """
-    Simple async token-bucket rate limiter: ensures no more than rate_per_sec
+    Simple async token-bucket rate limiter:
+    ensures no more than rate_per_sec
     requests are started per second, across all coroutines.
     """
 
@@ -40,7 +43,8 @@ class RateLimiter:
             elapsed = now - self._last
             delay = self._interval - elapsed
             if delay > 0:
-                await asyncio.sleep(delay)
+                jitter = random.uniform(0, 0.3)
+                await asyncio.sleep(delay + jitter)
             self._last = time.monotonic()
 
 
@@ -58,10 +62,13 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         _cookies (Dict[str, str]): Optional cookie jar for the session.
     """
 
-    def _init_session(
+    def is_async(self) -> Literal[True]:
+        return True
+
+    def __init__(
         self,
         config: RequesterConfig,
-        cookies: Optional[Dict[str, str]] = None,
+        cookies: dict[str, str] | None = None,
     ) -> None:
         """
         Initialize the async session with configuration.
@@ -71,26 +78,30 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         :param cookies: Optional initial cookies to set on the session.
         """
         self._config = config
-        self._timeout = config.timeout
         self._retry_times = config.retry_times
-        self._retry_interval = config.retry_interval
+        self._retry_interval = config.backoff_factor
+        self._timeout = config.timeout
+        self._max_rps = config.max_rps
+        self._max_connections = config.max_connections
+
         self._cookies = cookies or {}
         self._headers = DEFAULT_USER_HEADERS.copy()
-        self._session: Optional[ClientSession] = None
-        self._rate_limiter: Optional[RateLimiter] = None
+        self._session: ClientSession | None = None
+        self._rate_limiter: RateLimiter | None = None
+
+        self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
+
+        self._init_session()
 
-    async def _setup(self) -> None:
+    def _init_session(self) -> None:
         """
         Set up the aiohttp.ClientSession with timeout, connector, headers, and cookies.
         """
-        max_rps = getattr(self._config, "max_rps", None)
-        if max_rps is not None:
-            self._rate_limiter = RateLimiter(max_rps)
+        if self._max_rps is not None:
+            self._rate_limiter = RateLimiter(self._max_rps)
 
         timeout = ClientTimeout(total=self._timeout)
-        connector = TCPConnector(
-            limit_per_host=getattr(self._config, "max_connections", 10)
-        )
+        connector = TCPConnector(limit_per_host=self._max_connections)
         self._session = ClientSession(
             timeout=timeout,
             connector=connector,
@@ -98,7 +109,13 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
             cookies=self._cookies,
         )
 
-    async def login(self, max_retries: int = 3, manual_login: bool = False) -> bool:
+    async def login(
+        self,
+        username: str = "",
+        password: str = "",
+        manual_login: bool = False,
+        **kwargs: Any,
+    ) -> bool:
         """
         Attempt to log in asynchronously.
         Override in subclasses that require authentication.
@@ -112,7 +129,9 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
 
     @abc.abstractmethod
     async def get_book_info(
-        self, book_id: str, wait_time: Optional[float] = None
+        self,
+        book_id: str,
+        **kwargs: Any,
     ) -> str:
         """
         Fetch the raw HTML (or JSON) of the book info page asynchronously.
@@ -125,7 +144,10 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
 
     @abc.abstractmethod
     async def get_book_chapter(
-        self, book_id: str, chapter_id: str, wait_time: Optional[float] = None
+        self,
+        book_id: str,
+        chapter_id: str,
+        **kwargs: Any,
     ) -> str:
         """
         Fetch the raw HTML (or JSON) of a single chapter asynchronously.
@@ -137,7 +159,11 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         """
         ...
 
-    async def get_bookcase(self, wait_time: Optional[float] = None) -> str:
+    async def get_bookcase(
+        self,
+        page: int = 1,
+        **kwargs: Any,
+    ) -> str:
         """
         Optional: Retrieve the HTML content of the authenticated user's bookcase page.
         Subclasses that support user login/bookcase should override this.
@@ -159,17 +185,12 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         :return: The response body as text.
         :raises: aiohttp.ClientError on final failure.
         """
-        if self._session is None:
-            await self._setup()
-        if self._session is None:
-            raise RuntimeError("Session not initialized after setup")
-
         if self._rate_limiter:
             await self._rate_limiter.wait()
 
         for attempt in range(self._retry_times + 1):
             try:
-                async with self._session.get(url, **kwargs) as resp:
+                async with self.session.get(url, **kwargs) as resp:
                     resp.raise_for_status()
                     text: str = await resp.text()
                     return text
@@ -182,7 +203,10 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         raise RuntimeError("Unreachable code reached in fetch()")
 
     async def get(
-        self, url: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
+        self,
+        url: str,
+        params: dict[str, Any] | None = None,
+        **kwargs: Any,
     ) -> ClientResponse:
         """
         Send an HTTP GET request asynchronously.
@@ -193,20 +217,13 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         :return: aiohttp.ClientResponse object.
         :raises RuntimeError: If the session is not initialized.
         """
-        if self._session is None:
-            await self._setup()
-        if self._session is None:
-            raise RuntimeError("Session not initialized after setup")
-
-        if self._rate_limiter:
-            await self._rate_limiter.wait()
-        return await self._session.get(url, params=params, **kwargs)
+        return await self._request("GET", url, params=params, **kwargs)
 
     async def post(
         self,
         url: str,
-        data: Optional[Union[Dict[str, Any], bytes]] = None,
-        json: Optional[Dict[str, Any]] = None,
+        data: dict[str, Any] | bytes | None = None,
+        json: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> ClientResponse:
         """
@@ -219,14 +236,7 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         :return: aiohttp.ClientResponse object.
         :raises RuntimeError: If the session is not initialized.
         """
-        if self._session is None:
-            await self._setup()
-        if self._session is None:
-            raise RuntimeError("Session not initialized after setup")
-
-        if self._rate_limiter:
-            await self._rate_limiter.wait()
-        return await self._session.post(url, data=data, json=json, **kwargs)
+        return await self._request("POST", url, data=data, json=json, **kwargs)
 
     @property
     def session(self) -> ClientSession:
@@ -240,41 +250,106 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
         return self._session
 
     @property
-    def timeout(self) -> float:
-        """Return the default timeout setting."""
-        return self._timeout
+    def cookies(self) -> dict[str, str]:
+        """
+        Get the current session cookies.
 
-    @property
-    def retry_times(self) -> int:
-        """Return the maximum number of retry attempts."""
-        return self._retry_times
+        :return: A dict mapping cookie names to their values.
+        """
+        if self._session:
+            return {c.key: c.value for c in self._session.cookie_jar}
+        else:
+            return self._cookies
 
     @property
-    def retry_interval(self) -> float:
-        """Return the base interval (in seconds) between retries."""
-        return self._retry_interval
+    def headers(self) -> dict[str, str]:
+        """
+        Get the current session headers.
 
-    async def update_cookies(
-        self, cookies: Dict[str, str], overwrite: bool = True
-    ) -> None:
+        :return: A dict mapping header names to their values.
+        """
+        if self._session:
+            return dict(self._session.headers)
+        else:
+            return self._headers
+
+    def get_header(self, key: str, default: Any = None) -> Any:
         """
-        Update cookies for the current session and internal cache.
+        Retrieve a specific header value by name.
 
-        :param cookies: New cookies to merge.
-        :param overwrite: If True, replace existing; else, only set missing.
+        :param key: The header name to look up.
+        :param default: The value to return if the header is not present.
+        :return: The header value if present, else default.
         """
-        # update internal cache
-        if overwrite:
-            self._cookies.update({str(k): str(v) for k, v in cookies.items()})
+        if self._session:
+            return self._session.headers.get(key, default)
         else:
-            for k, v in cookies.items():
-                self._cookies.setdefault(str(k), str(v))
+            return self._headers.get(key, default)
+
+    def update_header(self, key: str, value: str) -> None:
+        """
+        Update or add a single header in the session.
+
+        :param key: The name of the header.
+        :param value: The value of the header.
+        """
+        self._headers[key] = value
+        if self._session:
+            self._session.headers[key] = value
+
+    def update_headers(self, headers: dict[str, str]) -> None:
+        """
+        Update or add multiple headers in the session.
+
+        :param headers: A dictionary of header key-value pairs.
+        """
+        self._headers.update(headers)
+        if self._session:
+            self._session.headers.update(headers)
+
+    def update_cookie(self, key: str, value: str) -> None:
+        """
+        Update or add a single cookie in the session.
+
+        :param key: The name of the cookie.
+        :param value: The value of the cookie.
+        """
+        self._cookies[key] = value
+        if self._session:
+            self._session.cookie_jar.update_cookies({key: value})
+
+    def update_cookies(
+        self,
+        cookies: dict[str, str],
+    ) -> None:
+        """
+        Update or add multiple cookies in the session.
+
+        :param cookies: A dictionary of cookie key-value pairs.
+        """
+        self._cookies.update(cookies)
+        if self._session:
+            self._session.cookie_jar.update_cookies(cookies)
 
-        # apply to live session
+    def clear_cookies(self) -> None:
+        """
+        Clear cookies from the session.
+        """
+        self._cookies = {}
         if self._session:
-            self._session.cookie_jar.update_cookies(self._cookies)
+            self._session.cookie_jar.clear()
+
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        **kwargs: Any,
+    ) -> ClientResponse:
+        if self._rate_limiter:
+            await self._rate_limiter.wait()
+        return await self.session.request(method, url, **kwargs)
 
-    async def shutdown(self) -> None:
+    async def close(self) -> None:
         """
         Shutdown and clean up the session. Closes connection pool.
         """
@@ -282,16 +357,48 @@ class BaseAsyncSession(AsyncRequesterProtocol, abc.ABC):
             await self._session.close()
             self._session = None
 
-    def __getstate__(self) -> Dict[str, Any]:
+    def sync_close(self) -> None:
+        """
+        Sync wrapper for closing the aiohttp session
+        when called from sync contexts.
+        """
+        if self._session:
+            try:
+                loop = asyncio.get_running_loop()
+                loop.create_task(self.close())
+            except RuntimeError:
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                loop.run_until_complete(self.close())
+                loop.close()
+
+    async def __aenter__(self) -> Self:
+        if self._session is None:
+            self._init_session()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        tb: types.TracebackType | None,
+    ) -> None:
+        await self.close()
+
+    def __del__(self) -> None:
+        self.sync_close()
+
+    def __getstate__(self) -> dict[str, Any]:
         """
         Prepare object state for serialization: remove unpickleable session.
         """
+        self.sync_close()
         state = self.__dict__.copy()
         state.pop("_session", None)
         state.pop("_rate_limiter", None)
         return state
 
-    def __setstate__(self, state: Dict[str, Any]) -> None:
+    def __setstate__(self, state: dict[str, Any]) -> None:
         """
         Restore object state. Session will be lazily reinitialized on next request.
         """