novel-downloader 1.1.0__py3-none-any.whl

novel_downloader/core/requesters/base_session.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters.base_session
+---------------------------------------------
+
+This module defines the BaseSession class, which provides basic HTTP
+request capabilities using the requests library. It maintains a
+persistent session and supports retries, headers, and timeout configurations.
+"""
+
+import abc
+from typing import Any, Dict, Optional, Union
+
+import requests
+from requests import Response, Session
+from requests.adapters import HTTPAdapter, Retry
+
+from novel_downloader.config.models import RequesterConfig
+from novel_downloader.core.interfaces import RequesterProtocol
+from novel_downloader.utils.constants import DEFAULT_USER_HEADERS
+
+
+class BaseSession(RequesterProtocol, abc.ABC):
+    """
+    BaseSession wraps basic HTTP operations using requests.Session,
+    supporting retry logic, timeout, and persistent connections.
+
+    Attributes:
+        _session (requests.Session): The persistent HTTP session.
+        _timeout (int): Timeout for each request in seconds.
+    """
+
+    def _init_session(
+        self, config: RequesterConfig, cookies: Optional[Dict[str, str]] = None
+    ) -> None:
+        """
+        Initialize the requests.Session with default headers and retry strategy.
+
+        :param config: Configuration object for session behavior
+                        (timeouts, retries, headers, etc.)
+        """
+        self._config = config
+        self._timeout = config.timeout
+        self._retry_times = config.retry_times
+        self._retry_interval = config.retry_interval
+        self._cookies = cookies or {}
+        self._headers = DEFAULT_USER_HEADERS
+        self._session: Optional[Session] = None
+
+        self._setup()
+
+    def _setup(self) -> None:
+        """
+        Set up the session with retry strategy and apply default headers.
+        """
+        self._session = requests.Session()
+
+        retry_strategy = Retry(
+            total=self._config.retry_times,
+            backoff_factor=self._config.retry_interval,
+            status_forcelist=[429, 500, 502, 503, 504],
+            allowed_methods=["HEAD", "GET", "OPTIONS"],
+        )
+
+        adapter = HTTPAdapter(max_retries=retry_strategy)
+        self._session.mount("http://", adapter)
+        self._session.mount("https://", adapter)
+        self._session.headers.update(self._headers)
+
+        if self._cookies:
+            self._session.cookies.update(self._cookies)
+
+    def login(self, max_retries: int = 3, manual_login: bool = False) -> bool:
+        """
+        Attempt to log in
+        """
+        raise NotImplementedError(
+            "Login is not supported by this session type. "
+            "Override login() in your subclass to enable it."
+        )
+
+    @abc.abstractmethod
+    def get_book_info(self, book_id: str, wait_time: Optional[int] = None) -> str:
+        """
+        Fetch the raw HTML (or JSON) of the book info page.
+
+        :param book_id: The book identifier.
+        :param wait_time: Base number of seconds to wait before returning content.
+        :return: The page content as a string.
+        """
+        ...
+
+    @abc.abstractmethod
+    def get_book_chapter(
+        self, book_id: str, chapter_id: str, wait_time: Optional[int] = None
+    ) -> str:
+        """
+        Fetch the raw HTML (or JSON) of a single chapter.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :param wait_time: Base number of seconds to wait before returning content.
+        :return: The chapter content as a string.
+        """
+        ...
+
+    def get_bookcase(self, wait_time: Optional[int] = None) -> str:
+        """
+        Optional: Retrieve the HTML content of the authenticated user's bookcase page.
+
+        Subclasses that support user login and bookcase retrieval should override this.
+
+        :param wait_time: Base number of seconds to wait before returning content.
+        :return: The HTML markup of the bookcase page.
+        :raises NotImplementedError: If the subclass does not implement.
+        """
+        raise NotImplementedError(
+            "Bookcase fetching is not supported by this session type. "
+            "Override get_bookcase() in your subclass to enable it."
+        )
+
+    def get(
+        self, url: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
+    ) -> Response:
+        """
+        Send a GET request.
+
+        :param url: The target URL.
+        :param params: Query parameters to include in the request.
+        :param kwargs: Additional arguments passed to requests.
+        :return: Response object from the GET request.
+        :raises RuntimeError: If the session is not initialized.
+        """
+        if not self._session:
+            raise RuntimeError("Session is not initialized or has been shut down.")
+        return self._session.get(url, params=params, timeout=self._timeout, **kwargs)
+
+    def post(
+        self,
+        url: str,
+        data: Optional[Union[Dict[str, Any], bytes]] = None,
+        json: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Response:
+        """
+        Send a POST request.
+
+        :param url: The target URL.
+        :param data: Form data to include in the request body.
+        :param json: JSON body to include in the request.
+        :param kwargs: Additional arguments passed to requests.
+        :return: Response object from the POST request.
+        :raises RuntimeError: If the session is not initialized.
+        """
+        if not self._session:
+            raise RuntimeError("Session is not initialized or has been shut down.")
+        return self._session.post(
+            url, data=data, json=json, timeout=self._timeout, **kwargs
+        )
+
+    @property
+    def session(self) -> Session:
+        """
+        Return the active requests.Session.
+
+        :raises RuntimeError: If the session is uninitialized or has been shut down.
+        """
+        if self._session is None:
+            raise RuntimeError("Session is not initialized or has been shut down.")
+        return self._session
+
+    @property
+    def timeout(self) -> int:
+        """Return the default timeout setting."""
+        return self._timeout
+
+    @property
+    def retry_times(self) -> int:
+        """Return the maximum number of retry attempts."""
+        return self._retry_times
+
+    @property
+    def retry_interval(self) -> float:
+        """Return the base interval (in seconds) between retries."""
+        return self._retry_interval
+
+    @property
+    def headers(self) -> Dict[str, str]:
+        """Return the default headers."""
+        if not self._session:
+            return {}
+        return {k: v for k, v in self._session.headers.items() if isinstance(v, str)}
+
+    def update_cookies(self, cookies: Dict[str, str], overwrite: bool = True) -> None:
+        """
+        Update cookies for the current session (if initialized) as well as for the
+        internal cache kept in ``self._cookies`` so that subsequent ``_setup`` calls
+        also see the latest values.
+        """
+        if not cookies:
+            return
+
+        if overwrite:
+            for k, v in cookies.items():
+                self._cookies[str(k)] = str(v)
+        else:
+            for k, v in cookies.items():
+                self._cookies.setdefault(str(k), str(v))
+
+        if self._session is not None:
+            self._session.cookies.update(self._cookies)
+
+    def shutdown(self) -> None:
+        """
+        Shutdown and clean up the session.
+
+        This closes the underlying connection pool and removes the session.
+        """
+        if self._session:
+            self._session.close()
+            self._session = None
+
+    def __getstate__(self) -> Dict[str, Any]:
+        """
+        Prepare object state for serialization.
+
+        Removes unpickleable session object.
+
+        :return: Serializable dict of the object state.
+        """
+        state = self.__dict__.copy()
+        state.pop("_session", None)
+        return state
+
+    def __setstate__(self, state: Dict[str, Any]) -> None:
+        """
+        Restore object state and reinitialize session.
+
+        :param state: Saved state dictionary.
+        """
+        self.__dict__.update(state)
+        self._setup()

novel_downloader/core/requesters/common_requester/__init__.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters.common_requester
+-------------------------------------------------
+
+This module provides the `CommonSession` class wrapper for common HTTP
+request operations to novel websites. It serves as a unified access
+point to import `CommonSession` without exposing lower-level modules.
+"""
+
+from .common_async_session import CommonAsyncSession
+from .common_session import CommonSession
+
+__all__ = [
+    "CommonAsyncSession",
+    "CommonSession",
+]

novel_downloader/core/requesters/common_requester/common_async_session.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters.common_requester.common_async_session
+----------------------------------------------------------------------
+
+This module defines a `CommonAsyncSession` class for handling HTTP requests
+to common novel sites **asynchronously**. It provides methods to retrieve
+raw book info pages and chapter contents using a flexible URL templating
+system defined by a site profile, with retry logic and random delays.
+"""
+
+import asyncio
+import random
+from typing import Dict, Optional
+
+from novel_downloader.config import RequesterConfig, SiteProfile
+from novel_downloader.core.requesters.base_async_session import BaseAsyncSession
+
+
+class CommonAsyncSession(BaseAsyncSession):
+    """
+    A common async session for handling site-specific HTTP requests.
+
+    :ivar _site: The unique identifier or name of the site.
+    :ivar _profile: Metadata and URL templates related to the site.
+    """
+
+    def __init__(
+        self,
+        config: RequesterConfig,
+        site: str,
+        profile: SiteProfile,
+        cookies: Optional[Dict[str, str]] = None,
+    ) -> None:
+        """
+        Initialize a CommonAsyncSession instance.
+
+        :param config: The RequesterConfig instance containing settings.
+        :param site: The identifier or domain of the target site.
+        :param profile: The site's metadata and URL templates.
+        :param cookies: Optional cookies to preload into the session.
+        """
+        self._init_session(config=config, cookies=cookies)
+        self._site = site
+        self._profile = profile
+
+    async def get_book_info(self, book_id: str, wait_time: Optional[int] = None) -> str:
+        """
+        Fetch the raw HTML of the book info page asynchronously.
+
+        Relies on BaseAsyncSession.fetch for retry logic, then sleeps with jitter.
+
+        :param book_id:   The book identifier.
+        :param wait_time: Base seconds to sleep (with 0.5-1.5x random factor).
+        :return:          The page content as a string.
+        """
+        url = self.book_info_url.format(book_id=book_id)
+        html = await self.fetch(url)
+        base = wait_time if wait_time is not None else self._config.wait_time
+        await asyncio.sleep(base * random.uniform(0.5, 1.5))
+        return html
+
+    async def get_book_chapter(
+        self, book_id: str, chapter_id: str, wait_time: Optional[int] = None
+    ) -> str:
+        """
+        Fetch the raw HTML of a single chapter asynchronously.
+
+        Relies on BaseAsyncSession.fetch for retry logic, then sleeps with jitter.
+
+        :param book_id:    The book identifier.
+        :param chapter_id: The chapter identifier.
+        :param wait_time:  Base seconds to sleep (with 0.5-1.5x random factor).
+        :return:           The chapter content as a string.
+        """
+        url = self.chapter_url.format(book_id=book_id, chapter_id=chapter_id)
+        html = await self.fetch(url)
+        base = wait_time if wait_time is not None else self._config.wait_time
+        await asyncio.sleep(base * random.uniform(0.5, 1.5))
+        return html
+
+    @property
+    def site(self) -> str:
+        """Return the site name."""
+        return self._site
+
+    @property
+    def book_info_url(self) -> str:
+        """Return the URL template for fetching book info."""
+        return self._profile["book_info_url"]
+
+    @property
+    def chapter_url(self) -> str:
+        """Return the URL template for fetching chapter content."""
+        return self._profile["chapter_url"]

novel_downloader/core/requesters/common_requester/common_session.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters.common_requester.common_session
+------------------------------------------------------------------
+
+This module defines a `CommonSession` class for handling HTTP requests
+to common novel sites. It provides methods to retrieve raw book
+information pages and chapter contents using a flexible URL templating
+system defined by a site profile.
+"""
+
+import time
+from typing import Dict, Optional
+
+from novel_downloader.config import RequesterConfig, SiteProfile
+from novel_downloader.utils.time_utils import sleep_with_random_delay
+
+from ..base_session import BaseSession
+
+
+class CommonSession(BaseSession):
+    """
+    A common session for handling site-specific HTTP requests.
+
+    :ivar _site: The unique identifier or name of the site.
+    :ivar _profile: Metadata and URL templates related to the site.
+    :ivar session: The HTTP session used to make requests.
+    """
+
+    def __init__(
+        self,
+        config: RequesterConfig,
+        site: str,
+        profile: SiteProfile,
+        cookies: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Initialize a CommonSession instance.
+
+        :param config: The RequesterConfig instance containing settings.
+        :param site: The identifier or domain of the target site.
+        :param profile: The site's metadata and URL templates.
+        :param cookies: Optional cookies to preload into the session.
+        """
+        self._init_session(config=config, cookies=cookies)
+        self._site = site
+        self._profile = profile
+
+    def get_book_info(self, book_id: str, wait_time: Optional[int] = None) -> str:
+        """
+        Fetch the raw HTML (or JSON) of the book info page.
+
+        :param book_id: The book identifier.
+        :param wait_time: Base number of seconds to wait before returning content.
+        :return: The page content as a string.
+        :raises requests.HTTPError: If the request returns an unsuccessful status code.
+        """
+        url = self.book_info_url.format(book_id=book_id)
+        base = wait_time if wait_time is not None else self._config.wait_time
+
+        for attempt in range(1, self.retry_times + 1):
+            try:
+                with self.session.get(url, timeout=self.timeout) as response:
+                    response.raise_for_status()
+                    content = response.text
+                sleep_with_random_delay(base)
+                return content
+            except Exception as e:
+                if attempt == self.retry_times:
+                    raise e  # 最后一次也失败了，抛出异常
+                else:
+                    time.sleep(self.retry_interval)
+                    continue
+        raise RuntimeError("Unexpected error: get_book_info failed without returning")
+
+    def get_book_chapter(
+        self, book_id: str, chapter_id: str, wait_time: Optional[int] = None
+    ) -> str:
+        """
+        Fetch the raw HTML (or JSON) of a single chapter.
+
+        :param book_id: The book identifier.
+        :param chapter_id: The chapter identifier.
+        :param wait_time: Base number of seconds to wait before returning content.
+        :return: The chapter content as a string.
+        :raises requests.HTTPError: If the request returns an unsuccessful status code.
+        """
+        url = self.chapter_url.format(book_id=book_id, chapter_id=chapter_id)
+        base = wait_time if wait_time is not None else self._config.wait_time
+
+        for attempt in range(1, self.retry_times + 1):
+            try:
+                with self.session.get(url, timeout=self.timeout) as response:
+                    response.raise_for_status()
+                    content = response.text
+                sleep_with_random_delay(base)
+                return content
+            except Exception as e:
+                if attempt == self.retry_times:
+                    raise e  # 最后一次也失败了，抛出异常
+                else:
+                    time.sleep(self.retry_interval)
+                    continue
+        raise RuntimeError(
+            "Unexpected error: get_book_chapter failed without returning"
+        )
+
+    @property
+    def site(self) -> str:
+        """Return the site name."""
+        return self._site
+
+    @property
+    def book_info_url(self) -> str:
+        """
+        Return the URL template for fetching book information.
+        """
+        return self._profile["book_info_url"]
+
+    @property
+    def chapter_url(self) -> str:
+        """
+        Return the URL template for fetching chapter information.
+        """
+        return self._profile["chapter_url"]

novel_downloader/core/requesters/qidian_requester/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+novel_downloader.core.requesters.qidian_requester
+-------------------------------------------------
+
+This package provides the implementation of the Qidian-specific requester logic.
+It contains modules for interacting with Qidian's website, including login,
+page navigation, and data retrieval using a browser-based automation approach.
+
+Modules:
+- qidian_browser: Implements the QidianBrowser class for automated browser control.
+- qidian_session: Implements the QidianSession class.
+"""
+
+from .qidian_broswer import QidianBrowser
+from .qidian_session import QidianSession
+
+__all__ = [
+    "QidianBrowser",
+    "QidianSession",
+]