scrapling 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

scrapling/engines/static.py

@@ -1,7 +1,7 @@
 from time import sleep as time_sleep
 from asyncio import sleep as asyncio_sleep
 
-from curl_cffi.requests.session import CurlError
+from curl_cffi.curl import CurlError
 from curl_cffi import CurlHttpVersion
 from curl_cffi.requests.impersonate import DEFAULT_CHROME
 from curl_cffi.requests import (
@@ -22,13 +22,14 @@ from scrapling.core._types import (
     Awaitable,
     List,
     Any,
+    cast,
 )
 
 from .toolbelt.custom import Response
 from .toolbelt.convertor import ResponseFactory
 from .toolbelt.fingerprints import generate_convincing_referer, generate_headers, __default_useragent__
 
-_UNSET = object()
+_UNSET: Any = object()
 
 
 class FetcherSession:
@@ -233,13 +234,12 @@ class FetcherSession:
         request_args: Dict[str, Any],
         max_retries: int,
         retry_delay: int,
-        selector_config: Optional[Dict] = None,
+        selector_config: Dict,
     ) -> Response:
         """
         Perform an HTTP request using the configured session.
 
         :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
-        :param url: Target URL for the request.
         :param request_args: Arguments to be passed to the session's `request()` method.
         :param max_retries: Maximum number of retries for the request.
         :param retry_delay: Number of seconds to wait between retries.
@@ -274,13 +274,12 @@ class FetcherSession:
         request_args: Dict[str, Any],
         max_retries: int,
         retry_delay: int,
-        selector_config: Optional[Dict] = None,
+        selector_config: Dict,
     ) -> Response:
         """
         Perform an HTTP request using the configured session.
 
         :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"]
-        :param url: Target URL for the request.
         :param request_args: Arguments to be passed to the session's `request()` method.
         :param max_retries: Maximum number of retries for the request.
         :param retry_delay: Number of seconds to wait between retries.
@@ -646,18 +645,420 @@ class FetcherSession:
 class FetcherClient(FetcherSession):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.__enter__ = None
-        self.__exit__ = None
-        self.__aenter__ = None
-        self.__aexit__ = None
-        self._curl_session = True
+        self.__enter__: Any = None
+        self.__exit__: Any = None
+        self.__aenter__: Any = None
+        self.__aexit__: Any = None
+        self._curl_session: Any = True
+
+    # Setting the correct return types for the type checking/autocompletion
+    def get(
+        self,
+        url: str,
+        params: Optional[Dict | List | Tuple] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Response:
+        return cast(
+            Response,
+            super().get(
+                url,
+                params,
+                headers,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def post(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Response:
+        return cast(
+            Response,
+            super().post(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def put(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Response:
+        return cast(
+            Response,
+            super().put(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def delete(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Response:
+        return cast(
+            Response,
+            super().delete(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
 
 
 class AsyncFetcherClient(FetcherSession):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.__enter__ = None
-        self.__exit__ = None
-        self.__aenter__ = None
-        self.__aexit__ = None
-        self._async_curl_session = True
+        self.__enter__: Any = None
+        self.__exit__: Any = None
+        self.__aenter__: Any = None
+        self.__aexit__: Any = None
+        self._async_curl_session: Any = True
+
+    # Setting the correct return types for the type checking/autocompletion
+    def get(
+        self,
+        url: str,
+        params: Optional[Dict | List | Tuple] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Awaitable[Response]:
+        return cast(
+            Awaitable[Response],
+            super().get(
+                url,
+                params,
+                headers,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def post(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Awaitable[Response]:
+        return cast(
+            Awaitable[Response],
+            super().post(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def put(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Awaitable[Response]:
+        return cast(
+            Awaitable[Response],
+            super().put(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )
+
+    def delete(
+        self,
+        url: str,
+        data: Optional[Dict | str] = None,
+        json: Optional[Dict | List] = None,
+        headers: Optional[Mapping[str, Optional[str]]] = _UNSET,
+        params: Optional[Dict | List | Tuple] = None,
+        cookies: Optional[CookieTypes] = None,
+        timeout: Optional[int | float] = _UNSET,
+        follow_redirects: Optional[bool] = _UNSET,
+        max_redirects: Optional[int] = _UNSET,
+        retries: Optional[int] = _UNSET,
+        retry_delay: Optional[int] = _UNSET,
+        proxies: Optional[ProxySpec] = _UNSET,
+        proxy: Optional[str] = _UNSET,
+        proxy_auth: Optional[Tuple[str, str]] = _UNSET,
+        auth: Optional[Tuple[str, str]] = None,
+        verify: Optional[bool] = _UNSET,
+        cert: Optional[str | Tuple[str, str]] = _UNSET,
+        impersonate: Optional[BrowserTypeLiteral] = _UNSET,
+        http3: Optional[bool] = _UNSET,
+        stealthy_headers: Optional[bool] = _UNSET,
+        **kwargs,
+    ) -> Awaitable[Response]:
+        return cast(
+            Awaitable[Response],
+            super().delete(
+                url,
+                data,
+                json,
+                headers,
+                params,
+                cookies,
+                timeout,
+                follow_redirects,
+                max_redirects,
+                retries,
+                retry_delay,
+                proxies,
+                proxy,
+                proxy_auth,
+                auth,
+                verify,
+                cert,
+                impersonate,
+                http3,
+                stealthy_headers,
+                **kwargs,
+            ),
+        )

scrapling/engines/toolbelt/navigation.py

@@ -4,7 +4,7 @@ Functions related to files and URLs
 
 from pathlib import Path
 from functools import lru_cache
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlparse
 
 from playwright.async_api import Route as async_Route
 from msgspec import Struct, structs, convert, ValidationError

scrapling/fetchers/__init__.py

@@ -0,0 +1,36 @@
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from scrapling.fetchers.requests import Fetcher, AsyncFetcher, FetcherSession
+    from scrapling.fetchers.chrome import DynamicFetcher, DynamicSession, AsyncDynamicSession
+    from scrapling.fetchers.firefox import StealthyFetcher, StealthySession, AsyncStealthySession
+
+
+# Lazy import mapping
+_LAZY_IMPORTS = {
+    "Fetcher": ("scrapling.fetchers.requests", "Fetcher"),
+    "AsyncFetcher": ("scrapling.fetchers.requests", "AsyncFetcher"),
+    "FetcherSession": ("scrapling.fetchers.requests", "FetcherSession"),
+    "DynamicFetcher": ("scrapling.fetchers.chrome", "DynamicFetcher"),
+    "DynamicSession": ("scrapling.fetchers.chrome", "DynamicSession"),
+    "AsyncDynamicSession": ("scrapling.fetchers.chrome", "AsyncDynamicSession"),
+    "StealthyFetcher": ("scrapling.fetchers.firefox", "StealthyFetcher"),
+    "StealthySession": ("scrapling.fetchers.firefox", "StealthySession"),
+    "AsyncStealthySession": ("scrapling.fetchers.firefox", "AsyncStealthySession"),
+}
+
+__all__ = ["Fetcher", "AsyncFetcher", "StealthyFetcher", "DynamicFetcher"]
+
+
+def __getattr__(name: str) -> Any:
+    if name in _LAZY_IMPORTS:
+        module_path, class_name = _LAZY_IMPORTS[name]
+        module = __import__(module_path, fromlist=[class_name])
+        return getattr(module, class_name)
+    else:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    """Support for dir() and autocomplete."""
+    return sorted(list(_LAZY_IMPORTS.keys()))