ultimate-sitemap-parser 1.0.0rc1__py3-none-any.whl

usp/web_client/abstract_client.py

@@ -0,0 +1,189 @@
+"""Abstract web client class."""
+
+import abc
+from http import HTTPStatus
+from typing import Optional
+
+RETRYABLE_HTTP_STATUS_CODES = {
+    # Some servers return "400 Bad Request" initially but upon retry start working again, no idea why
+    int(HTTPStatus.BAD_REQUEST),
+    # If we timed out requesting stuff, we can just try again
+    int(HTTPStatus.REQUEST_TIMEOUT),
+    # If we got rate limited, it makes sense to wait a bit
+    int(HTTPStatus.TOO_MANY_REQUESTS),
+    # Server might be just fine on a subsequent attempt
+    int(HTTPStatus.INTERNAL_SERVER_ERROR),
+    # Upstream might reappear on a retry
+    int(HTTPStatus.BAD_GATEWAY),
+    # Service might become available again on a retry
+    int(HTTPStatus.SERVICE_UNAVAILABLE),
+    # Upstream might reappear on a retry
+    int(HTTPStatus.GATEWAY_TIMEOUT),
+    # (unofficial) 509 Bandwidth Limit Exceeded (Apache Web Server/cPanel)
+    509,
+    # (unofficial) 598 Network read timeout error
+    598,
+    # (unofficial, nginx) 499 Client Closed Request
+    499,
+    # (unofficial, Cloudflare) 520 Unknown Error
+    520,
+    # (unofficial, Cloudflare) 521 Web Server Is Down
+    521,
+    # (unofficial, Cloudflare) 522 Connection Timed Out
+    522,
+    # (unofficial, Cloudflare) 523 Origin Is Unreachable
+    523,
+    # (unofficial, Cloudflare) 524 A Timeout Occurred
+    524,
+    # (unofficial, Cloudflare) 525 SSL Handshake Failed
+    525,
+    # (unofficial, Cloudflare) 526 Invalid SSL Certificate
+    526,
+    # (unofficial, Cloudflare) 527 Railgun Error
+    527,
+    # (unofficial, Cloudflare) 530 Origin DNS Error
+    530,
+}
+"""HTTP status codes on which a request should be retried."""
+
+
+class AbstractWebClientResponse(metaclass=abc.ABCMeta):
+    """
+    Abstract response.
+    """
+
+    pass
+
+
+class AbstractWebClientSuccessResponse(
+    AbstractWebClientResponse, metaclass=abc.ABCMeta
+):
+    """
+    Successful response.
+    """
+
+    @abc.abstractmethod
+    def status_code(self) -> int:
+        """
+        Return HTTP status code of the response.
+
+        :return: HTTP status code of the response, e.g. 200.
+        """
+        raise NotImplementedError("Abstract method.")
+
+    @abc.abstractmethod
+    def status_message(self) -> str:
+        """
+        Return HTTP status message of the response.
+
+        :return: HTTP status message of the response, e.g. "OK".
+        """
+        raise NotImplementedError("Abstract method.")
+
+    @abc.abstractmethod
+    def header(self, case_insensitive_name: str) -> Optional[str]:
+        """
+        Return HTTP header value for a given case-insensitive name, or None if such header wasn't set.
+
+        :param case_insensitive_name: HTTP header's name, e.g. "Content-Type".
+        :return: HTTP header's value, or None if it was unset.
+        """
+        raise NotImplementedError("Abstract method.")
+
+    @abc.abstractmethod
+    def raw_data(self) -> bytes:
+        """
+        Return encoded raw data of the response.
+
+        :return: Encoded raw data of the response.
+        """
+        raise NotImplementedError("Abstract method.")
+
+
+class WebClientErrorResponse(AbstractWebClientResponse, metaclass=abc.ABCMeta):
+    """
+    Error response.
+    """
+
+    __slots__ = [
+        "_message",
+        "_retryable",
+    ]
+
+    def __init__(self, message: str, retryable: bool):
+        """
+        Constructor.
+
+        :param message: Message describing what went wrong.
+        :param retryable: True if the request should be retried.
+        """
+        super().__init__()
+        self._message = message
+        self._retryable = retryable
+
+    def message(self) -> str:
+        """
+        Return message describing what went wrong.
+
+        :return: Message describing what went wrong.
+        """
+        return self._message
+
+    def retryable(self) -> bool:
+        """
+        Return True if request should be retried.
+
+        :return: True if request should be retried.
+        """
+        return self._retryable
+
+
+class AbstractWebClient(metaclass=abc.ABCMeta):
+    """
+    Abstract web client to be used by the sitemap fetcher.
+    """
+
+    @abc.abstractmethod
+    def set_max_response_data_length(
+        self, max_response_data_length: Optional[int]
+    ) -> None:
+        """
+        Set the maximum number of bytes that the web client will fetch.
+
+        :param max_response_data_length: Maximum number of bytes that the web client will fetch, or None to fetch all.
+        """
+        raise NotImplementedError("Abstract method.")
+
+    @abc.abstractmethod
+    def get(self, url: str) -> AbstractWebClientResponse:
+        """
+        Fetch a URL and return a response.
+
+        Method shouldn't throw exceptions on connection errors (including timeouts); instead, such errors should be
+        reported via Response object.
+
+        :param url: URL to fetch.
+        :return: Response object.
+        """
+        raise NotImplementedError("Abstract method.")
+
+
+class NoWebClientException(Exception):
+    """Error indicating this web client cannot fetch pages."""
+
+    pass
+
+
+class LocalWebClient(AbstractWebClient):
+    """Dummy web client which is a valid implementation but errors if called.
+
+    Used for local parsing
+    """
+
+    def set_max_response_data_length(
+        self, max_response_data_length: Optional[int]
+    ) -> None:
+        pass
+
+    def get(self, url: str) -> AbstractWebClientResponse:
+        raise NoWebClientException

usp/web_client/requests_client.py

@@ -0,0 +1,150 @@
+"""Implementation of :mod:`usp.web_client.abstract_client` with Requests."""
+
+from http import HTTPStatus
+from typing import Optional, Dict, Tuple, Union
+
+import requests
+
+from .abstract_client import (
+    AbstractWebClient,
+    AbstractWebClientResponse,
+    AbstractWebClientSuccessResponse,
+    WebClientErrorResponse,
+    RETRYABLE_HTTP_STATUS_CODES,
+)
+from usp import __version__
+
+
+class RequestsWebClientSuccessResponse(AbstractWebClientSuccessResponse):
+    """
+    requests-based successful response.
+    """
+
+    __slots__ = [
+        "__requests_response",
+        "__max_response_data_length",
+    ]
+
+    def __init__(
+        self,
+        requests_response: requests.Response,
+        max_response_data_length: Optional[int] = None,
+    ):
+        """
+        :param requests_response: Response data
+        :param max_response_data_length: Maximum data length, or ``None`` to not restrict.
+        """
+        self.__requests_response = requests_response
+        self.__max_response_data_length = max_response_data_length
+
+    def status_code(self) -> int:
+        return int(self.__requests_response.status_code)
+
+    def status_message(self) -> str:
+        message = self.__requests_response.reason
+        if not message:
+            message = HTTPStatus(self.status_code()).phrase
+        return message
+
+    def header(self, case_insensitive_name: str) -> Optional[str]:
+        return self.__requests_response.headers.get(case_insensitive_name.lower(), None)
+
+    def raw_data(self) -> bytes:
+        if self.__max_response_data_length:
+            data = self.__requests_response.content[: self.__max_response_data_length]
+        else:
+            data = self.__requests_response.content
+
+        return data
+
+
+class RequestsWebClientErrorResponse(WebClientErrorResponse):
+    """
+    Error response from the Requests parser.
+    """
+
+    pass
+
+
+class RequestsWebClient(AbstractWebClient):
+    """requests-based web client to be used by the sitemap fetcher."""
+
+    __USER_AGENT = f"ultimate_sitemap_parser/{__version__}"
+
+    __HTTP_REQUEST_TIMEOUT = 60
+    """
+    HTTP request timeout.
+
+    Some webservers might be generating huge sitemaps on the fly, so this is why it's rather big.
+    """
+
+    __slots__ = ["__max_response_data_length", "__timeout", "__proxies", "__verify"]
+
+    def __init__(self, verify=True):
+        """
+        :param verify: whether certificates should be verified for HTTPS requests.
+        """
+        self.__max_response_data_length = None
+        self.__timeout = self.__HTTP_REQUEST_TIMEOUT
+        self.__proxies = {}
+        self.__verify = verify
+
+    def set_timeout(self, timeout: Union[int, Tuple[int, int], None]) -> None:
+        """Set HTTP request timeout.
+
+        See also: `Requests timeout docs <https://requests.readthedocs.io/en/latest/user/advanced/#timeouts>`__
+
+        :param timeout: An integer to use as both the connect and read timeouts,
+            or a tuple to specify them individually, or None for no timeout
+        """
+        # Used mostly for testing
+        self.__timeout = timeout
+
+    def set_proxies(self, proxies: Dict[str, str]) -> None:
+        """
+        Set a proxy for the request.
+
+        :param proxies: Proxy definition where the keys are schemes ("http" or "https") and values are the proxy address.
+            Example: ``{'http': 'http://user:pass@10.10.1.10:3128/'}, or an empty dict to disable proxy.``
+        """
+        # Used mostly for testing
+        self.__proxies = proxies
+
+    def set_max_response_data_length(self, max_response_data_length: int) -> None:
+        self.__max_response_data_length = max_response_data_length
+
+    def get(self, url: str) -> AbstractWebClientResponse:
+        try:
+            response = requests.get(
+                url,
+                timeout=self.__timeout,
+                stream=True,
+                headers={"User-Agent": self.__USER_AGENT},
+                proxies=self.__proxies,
+                verify=self.__verify,
+            )
+        except requests.exceptions.Timeout as ex:
+            # Retryable timeouts
+            return RequestsWebClientErrorResponse(message=str(ex), retryable=True)
+
+        except requests.exceptions.RequestException as ex:
+            # Other errors, e.g. redirect loops
+            return RequestsWebClientErrorResponse(message=str(ex), retryable=False)
+
+        else:
+            if 200 <= response.status_code < 300:
+                return RequestsWebClientSuccessResponse(
+                    requests_response=response,
+                    max_response_data_length=self.__max_response_data_length,
+                )
+            else:
+                message = f"{response.status_code} {response.reason}"
+
+                if response.status_code in RETRYABLE_HTTP_STATUS_CODES:
+                    return RequestsWebClientErrorResponse(
+                        message=message, retryable=True
+                    )
+                else:
+                    return RequestsWebClientErrorResponse(
+                        message=message, retryable=False
+                    )