scrape-do-python 0.1.0__py3-none-any.whl

scrape_do/constants.py

@@ -0,0 +1,84 @@
+"""
+Defines constants with valid parameter values expected by the Scrape.do API
+
+Attributes:
+    _SUPER_SUPPORTED_COUNTRIES (set[str]): The complete list of ISO 3166-1
+        alpha-2 country codes supported when `super=True`.
+
+    _DATACENTER_SUPPORTED_COUNTRIES (set[str]): The restricted list of ISO
+        3166-1 alpha-2 country codes supported when `super=False`.
+
+    _ZIPCODE_FORMATS (dict[str, re.Pattern]): Pre-compiled regular expressions
+        mapping lowercase country codes to their strict regional postal code
+        formats.
+
+    _SUPER_ONLY_COUNTRIES (set[str]): ISO 3166-1 alpha-2 country codes
+        supported only when `super=True`
+
+    _ZIPCODE_ALLOWED_COUNTRIES (set[str]): Set of country codes for which the
+        `postal_code` parameter is allowed
+
+    _ZIPCODE_NOT_ALLOWED_COUNTRIES (set[str]): Set of country codes for which
+        the `postal_code` parameter is not allowed
+"""
+
+import re
+
+
+_SUPER_SUPPORTED_COUNTRIES = {
+    "ad", "ae", "af", "ag", "al", "am", "ao", "ar", "as", "at",
+    "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh",
+    "bi", "bj", "bm", "bn", "bo", "br", "bs", "bt", "bw", "by",
+    "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn",
+    "co", "cr", "cu", "cv", "cy", "cz", "de", "dj", "dk", "dm",
+    "do", "dz", "ec", "ee", "eg", "er", "es", "et", "fi", "fj",
+    "fm", "fr", "ga", "gb", "gd", "ge", "gh", "gi", "gm", "gn",
+    "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht",
+    "hu", "id", "ie", "il", "in", "iq", "ir", "is", "it", "jm",
+    "jo", "jp", "ke", "kg", "kh", "ki", "km", "kn", "kp", "kr",
+    "kw", "ky", "kz", "la", "lb", "lc", "li", "lk", "lr", "ls",
+    "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mg", "mh",
+    "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv",
+    "mw", "mx", "my", "mz", "na", "ne", "ng", "ni", "nl", "no",
+    "np", "nr", "nz", "om", "pa", "pe", "pg", "ph", "pk", "pl",
+    "pr", "pt", "pw", "py", "qa", "ro", "rs", "ru", "rw", "sa",
+    "sb", "sc", "sd", "se", "sg", "si", "sk", "sl", "sn", "so",
+    "sr", "ss", "st", "sv", "sy", "sz", "td", "tg", "th", "tj",
+    "tl", "tm", "tn", "to", "tr", "tt", "tv", "tw", "tz", "ua",
+    "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "vu",
+    "ws", "ye", "za", "zm", "zw"
+    }
+
+
+_DATACENTER_SUPPORTED_COUNTRIES = {
+    "ae", "al", "ar", "at", "au", "br", "ca", "ch", "cl", "cn",
+    "cr", "cy", "cz", "de", "dk", "ee", "eg", "es", "fi", "fr",
+    "gb", "gr", "hr", "ie", "it", "jp", "lt", "lv", "mt", "nl",
+    "no", "pk", "pl", "pt", "ro", "rs", "ru", "se", "sg", "si",
+    "sk", "tr", "ua", "us", "za"
+    }
+
+_ZIPCODE_FORMATS = {
+    "us": re.compile(r"^\d{5}$"),
+    "gb": re.compile(r"^[A-Z0-9\s]{2,8}$", re.IGNORECASE),
+    "de": re.compile(r"^\d{5}$"),
+    "fr": re.compile(r"^\d{5}$"),
+    "ca": re.compile(r"^[A-Z]\d[A-Z]\s?\d[A-Z]\d$", re.IGNORECASE),
+    "au": re.compile(r"^\d{4}$"),
+    "in": re.compile(r"^\d{6}$"),
+    "nl": re.compile(r"^\d{4}[A-Z]{2}$", re.IGNORECASE),
+    "it": re.compile(r"^\d{5}$"),
+    "es": re.compile(r"^\d{5}$"),
+    "br": re.compile(r"^(\d{5}|\d{8})$"),
+    "jp": re.compile(r"^\d{3}-?\d{4}$")
+    }
+
+_SUPER_ONLY_COUNTRIES = {c for c in _SUPER_SUPPORTED_COUNTRIES
+                         if c not in _DATACENTER_SUPPORTED_COUNTRIES
+                         }
+
+_ZIPCODE_ALLOWED_COUNTRIES = set(_ZIPCODE_FORMATS.keys())
+
+_ZIPCODE_NOT_ALLOWED_COUNTRIES = {c for c in _SUPER_SUPPORTED_COUNTRIES
+                                  if c not in _ZIPCODE_ALLOWED_COUNTRIES
+                                  }

scrape_do/exceptions.py

@@ -0,0 +1,238 @@
+"""Custom exception hierarchy and network error routing for the Scrape.do SDK.
+
+Dynamically parses API failures, distinguishes proxy infrastructure errors
+from target website blocks, and exposes programmatic flags for retry
+strategies.
+"""
+
+from __future__ import annotations
+import httpx
+from typing import Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .models import PreparedScrapeDoRequest, ScrapeDoResponse
+
+
+class ScrapeDoError(Exception):
+    """The base exception for all errors raised by the SDK.
+
+    Catching this exception guarantees that any error originating strictly
+    from the SDK or the proxy network is handled.
+
+    Args:
+        message (str): Error message to be displayed
+        request (Optional[PreparedScrapeDoRequest]): Object containing the
+            request's information if it exists, otherwise `None`.
+        response (Optional[ScrapeDoResponse]): Object containing the response's
+            information if it exists, otherwise `None`
+    """
+    def __init__(
+        self,
+        message: str,
+        request: Optional[PreparedScrapeDoRequest] = None,
+        response: Optional[ScrapeDoResponse] = None
+    ):
+        super().__init__(message)
+        self.message = message
+        self.request = request
+        self.response = response
+        if response is not None:
+            self.status_code = response.target_status_code
+
+
+class APIConnectionError(ScrapeDoError):
+    """Raised when the SDK fails to connect to the Scrape.do gateway entirely.
+
+    This indicates a network-level failure such as DNS resolution issues,
+    local internet outages, or hard socket timeouts.
+    """
+
+
+class TargetError(ScrapeDoError):
+    """Raised when the Scrape.do proxy connects, but the target website fails.
+
+    This exception is triggered when `transparent_response=True` is used,
+    explicitly flagging that the destination URL returned a non-2xx status
+    code.
+
+    Args:
+        message (str): The raw response body or error message from the target.
+        target_status_code (int): The HTTP status code returned by the target
+            website.
+        raw_response (httpx.Response): The raw HTTP response object.
+        request (Optional[PreparedScrapeDoRequest]): Object containing the
+            request's information if it exists, otherwise `None`.
+        response (Optional[ScrapeDoResponse]): Object containing the response's
+            information if it exists, otherwise `None`
+    """
+
+    def __init__(
+        self,
+        message: str,
+        target_status_code: int,
+        raw_response: httpx.Response,
+        request: Optional[PreparedScrapeDoRequest] = None,
+        response: Optional[ScrapeDoResponse] = None
+    ):
+        self.target_status_code = target_status_code
+        self.raw_response = raw_response
+        super().__init__(
+            f"Target website returned status {target_status_code}: {message}",
+            request,
+            response
+            )
+
+    @property
+    def is_waf_block(self) -> bool:
+        """
+        Programmatic flag to identify if the target website blocked the proxy.
+
+        Returns:
+            `True` if status code is either `401` or `403`, `False` otherwise
+        """
+        return self.target_status_code in (401, 403)
+
+    @property
+    def is_throttled(self) -> bool:
+        """
+        Programmatic flag to identify target-level rate limiting.
+
+        Returns:
+            `True` if status code is `429`, `False` otherwise
+        """
+        return self.target_status_code == 429
+
+
+class AuthenticationThrottleError(ScrapeDoError):
+    """Raised when high-frequency invalid requests trigger an authentication
+    ban.
+
+    Args:
+        raw_response (httpx.Response): The raw HTTP response object.
+        request (Optional[PreparedScrapeDoRequest]): Object containing the
+            request's information if it exists, otherwise `None`.
+        response (Optional[ScrapeDoResponse]): Object containing the response's
+            information if it exists, otherwise `None`
+    """
+    def __init__(
+        self,
+        raw_response: httpx.Response,
+        request: Optional[PreparedScrapeDoRequest] = None,
+        response: Optional[ScrapeDoResponse] = None
+    ):
+        msg = ("Your request has been temporarily throttled by the"
+               "authentication server."
+               )
+        self.raw_response = raw_response
+
+        super().__init__(msg, request, response)
+
+
+class APIResponseError(ScrapeDoError):
+    """Dynamically parses and represents a Scrape.do API infrastructure error.
+
+    This acts as the base exception for all non-2xx HTTP responses returned
+    by the Scrape.do gateway. It parses the JSON payloads to extract
+    human-readable error messages.
+
+    Args:
+        raw_response (httpx.Response): The raw HTTP response object.
+        request (Optional[PreparedScrapeDoRequest]): Object containing the
+            request's information if it exists, otherwise `None`.
+        response (Optional[ScrapeDoResponse]): Object containing the response's
+            information if it exists, otherwise `None`
+    """
+
+    def __init__(
+        self,
+        raw_response: httpx.Response,
+        request: Optional[PreparedScrapeDoRequest] = None,
+        response: Optional[ScrapeDoResponse] = None
+    ):
+        self.raw_response = raw_response
+        self.raw_status_code = raw_response.status_code
+        self.message = f"Unknown API Error. Body: {raw_response.text}"
+
+        # Attempt to parse known JSON keys
+        try:
+            data = raw_response.json()
+            for key in ("detail",
+                        "Error",
+                        "errorMessage",
+                        "message",
+                        "Message"
+                        ):
+                if key in data and isinstance(data[key], str):
+                    self.message = data[key]
+                    break
+
+        except ValueError:
+            pass
+
+        super().__init__(
+            (
+                f"API returned an error."
+                f"Status: {self.raw_status_code} | Message: {self.message}"
+                ),
+            request,
+            response
+            )
+
+
+# --- Specific API Response Subclasses ---
+
+class AuthenticationError(APIResponseError):
+    """Raised when the API returns an HTTP 401 (Unauthorized).
+
+    Indicates that the provided API token is missing or invalid.
+    """
+    pass
+
+
+class BadRequestError(APIResponseError):
+    """Raised when the API returns an HTTP 400 (Bad Request).
+
+    Indicates that the Scrape.do servers rejected the request configuration.
+    """
+    pass
+
+
+class RateLimitError(APIResponseError):
+    """Raised when the API returns an HTTP 429 (Too Many Requests).
+
+    Indicates that the account has exceeded its concurrent request limit.
+    """
+    pass
+
+
+class ServerError(APIResponseError):
+    """Raised when the API returns an HTTP 500+ status code.
+
+    Indicates a gateway failure or proxy pool outage.
+    """
+    pass
+
+
+class RotatedSessionError(ScrapeDoError):
+    """
+    Raised when a user-defined `session_validator` determines that the target
+    website's state has been lost (e.g., logged out, CAPTCHA triggered),
+    indicating that Scrape.do silently rotated the proxy exit node.
+
+    Args:
+        message (str): Error message to be displayed
+        raw_response (httpx.Response): The raw HTTP response object.
+        request (PreparedScrapeDoRequest): Object containing the
+            request's information
+        response (Optional[ScrapeDoResponse]): Object containing the response's
+            information
+    """
+    def __init__(
+        self,
+        message: str,
+        raw_response: httpx.Response,
+        request: PreparedScrapeDoRequest,
+        response: ScrapeDoResponse
+    ):
+        self.raw_response = raw_response
+        super().__init__(message, request, response)

scrape_do/models/__init__.py

@@ -0,0 +1,79 @@
+"""Public API for the data models and Scrape.do API contracts
+
+Aggregates domain models into a unified namespace to expose all necessary type
+hints, browser actions, and configuration contracts required to interact with
+the API.
+"""
+
+from __future__ import annotations
+
+from .browser_actions import (
+    ClickAction,
+    WaitAction,
+    WaitSelectorAction,
+    ScrollXAction,
+    ScrollYAction,
+    ScrollToAction,
+    FillAction,
+    ExecuteAction,
+    ScreenShotAction,
+    WaitForRequestCompletionAction,
+    BrowserAction
+    )
+from .enums import (
+    RegionCodeType,
+    WaitUntilType,
+    DeviceType,
+    OutputType,
+    HttpMethod,
+    PayloadType
+    )
+from .request import (
+    PreparedScrapeDoRequest
+)
+from .parameters import (
+    RequestParameters,
+    RequestParametersDict
+)
+from .response import (
+    ScrapeDoNetworkRequest,
+    ScrapeDoWebSocketFrame,
+    ScrapeDoWebSocketEvent,
+    ScrapeDoWebsocketRequest,
+    ScrapeDoActionResult,
+    ScrapeDoScreenshot,
+    ScrapeDoFrame,
+    ScrapeDoResponse
+    )
+
+
+__all__ = [
+    "ClickAction",
+    "WaitAction",
+    "WaitSelectorAction",
+    "ScrollXAction",
+    "ScrollYAction",
+    "ScrollToAction",
+    "FillAction",
+    "ExecuteAction",
+    "ScreenShotAction",
+    "WaitForRequestCompletionAction",
+    "BrowserAction",
+    "RegionCodeType",
+    "WaitUntilType",
+    "DeviceType",
+    "OutputType",
+    "HttpMethod",
+    "PayloadType",
+    "RequestParametersDict",
+    "RequestParameters",
+    "PreparedScrapeDoRequest",
+    "ScrapeDoNetworkRequest",
+    "ScrapeDoWebSocketFrame",
+    "ScrapeDoWebSocketEvent",
+    "ScrapeDoWebsocketRequest",
+    "ScrapeDoActionResult",
+    "ScrapeDoScreenshot",
+    "ScrapeDoFrame",
+    "ScrapeDoResponse"
+    ]