ultimate-sitemap-parser 1.4.0__tar.gz → 1.6.0__tar.gz

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ultimate-sitemap-parser
-Version: 1.4.0
+Version: 1.6.0
 Summary: A performant library for parsing and crawling sitemaps
 License: GPL-3.0-or-later
 Keywords: sitemap,crawler,indexing,xml,rss,atom,google news

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ultimate-sitemap-parser"
-version = "1.4.0"
+version = "1.6.0"
 description = "A performant library for parsing and crawling sitemaps"
 authors = [
     { name = "Linas Valiukas", email = "linas@media.mit.edu"},
@@ -103,4 +103,7 @@ select = [
 junit_suite_name = "ultimate-sitemap-parser"
 junit_duration_report = "call"
 log_cli = true
-log_cli_level = "DEBUG"
+log_cli_level = "DEBUG"
+filterwarnings = [
+    "error::pytest.PytestUnraisableExceptionWarning"
+]

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/usp/fetch_parse.py

@@ -17,6 +17,8 @@ from typing import Dict, Optional, Set
 
 from .exceptions import SitemapException, SitemapXMLParsingException
 from .helpers import (
+    RecurseCallbackType,
+    RecurseListCallbackType,
     get_url_retry_on_client_errors,
     html_unescape_strip,
     is_http_url,
@@ -77,6 +79,8 @@ class SitemapFetcher:
         "_web_client",
         "_parent_urls",
         "_quiet_404",
+        "_recurse_callback",
+        "_recurse_list_callback",
     ]
 
     def __init__(
@@ -86,6 +90,8 @@ class SitemapFetcher:
         web_client: Optional[AbstractWebClient] = None,
         parent_urls: Optional[Set[str]] = None,
         quiet_404: bool = False,
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
     ):
         """
 
@@ -94,6 +100,8 @@ class SitemapFetcher:
         :param web_client: Web client to use. If ``None``, a :class:`~.RequestsWebClient` will be used.
         :param parent_urls: Set of parent URLs that led to this sitemap.
         :param quiet_404: Whether 404 errors are expected and should be logged at a reduced level, useful for speculative fetching of known URLs.
+        :param recurse_callback: Optional callback to filter out a sub-sitemap. See :data:`~.RecurseCallbackType`.
+        :param recurse_list_callback: Optional callback to filter the list of sub-sitemaps. See :data:`~.RecurseListCallbackType`.
 
         :raises SitemapException: If the maximum recursion depth is exceeded.
         :raises SitemapException: If the URL is in the parent URLs set.
@@ -128,6 +136,9 @@ class SitemapFetcher:
         self._parent_urls = parent_urls or set()
         self._quiet_404 = quiet_404
 
+        self._recurse_callback = recurse_callback
+        self._recurse_list_callback = recurse_list_callback
+
     def _fetch(self) -> AbstractWebClientResponse:
         log.info(f"Fetching level {self._recursion_level} sitemap from {self._url}...")
         response = get_url_retry_on_client_errors(
@@ -173,6 +184,8 @@ class SitemapFetcher:
                 recursion_level=self._recursion_level,
                 web_client=self._web_client,
                 parent_urls=self._parent_urls,
+                recurse_callback=self._recurse_callback,
+                recurse_list_callback=self._recurse_list_callback,
             )
 
         else:
@@ -184,6 +197,8 @@ class SitemapFetcher:
                     recursion_level=self._recursion_level,
                     web_client=self._web_client,
                     parent_urls=self._parent_urls,
+                    recurse_callback=self._recurse_callback,
+                    recurse_list_callback=self._recurse_list_callback,
                 )
             else:
                 parser = PlainTextSitemapParser(
@@ -234,6 +249,8 @@ class AbstractSitemapParser(metaclass=abc.ABCMeta):
         "_web_client",
         "_recursion_level",
         "_parent_urls",
+        "_recurse_callback",
+        "_recurse_list_callback",
     ]
 
     def __init__(
@@ -243,6 +260,8 @@ class AbstractSitemapParser(metaclass=abc.ABCMeta):
         recursion_level: int,
         web_client: AbstractWebClient,
         parent_urls: Set[str],
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
     ):
         self._url = url
         self._content = content
@@ -250,6 +269,16 @@ class AbstractSitemapParser(metaclass=abc.ABCMeta):
         self._web_client = web_client
         self._parent_urls = parent_urls
 
+        if recurse_callback is None:  # Always allow child recursion
+            self._recurse_callback = lambda url, level, parent_urls: True
+        else:
+            self._recurse_callback = recurse_callback
+
+        if recurse_list_callback is None:  # Always allow child recursion
+            self._recurse_list_callback = lambda urls, level, parent_urls: urls
+        else:
+            self._recurse_list_callback = recurse_list_callback
+
     @abc.abstractmethod
     def sitemap(self) -> AbstractSitemap:
         """
@@ -270,6 +299,8 @@ class IndexRobotsTxtSitemapParser(AbstractSitemapParser):
         recursion_level: int,
         web_client: AbstractWebClient,
         parent_urls: Set[str],
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
     ):
         super().__init__(
             url=url,
@@ -277,6 +308,8 @@ class IndexRobotsTxtSitemapParser(AbstractSitemapParser):
             recursion_level=recursion_level,
             web_client=web_client,
             parent_urls=parent_urls,
+            recurse_callback=recurse_callback,
+            recurse_list_callback=recurse_list_callback,
         )
 
         if not self._url.endswith("/robots.txt"):
@@ -304,16 +337,27 @@ class IndexRobotsTxtSitemapParser(AbstractSitemapParser):
                     )
 
         sub_sitemaps = []
+        parent_urls = self._parent_urls | {self._url}
 
-        for sitemap_url in sitemap_urls.keys():
+        filtered_sitemap_urls = self._recurse_list_callback(
+            list(sitemap_urls.keys()), self._recursion_level, parent_urls
+        )
+        for sitemap_url in filtered_sitemap_urls:
             try:
-                fetcher = SitemapFetcher(
-                    url=sitemap_url,
-                    recursion_level=self._recursion_level + 1,
-                    web_client=self._web_client,
-                    parent_urls=self._parent_urls | {self._url},
-                )
-                fetched_sitemap = fetcher.sitemap()
+                if self._recurse_callback(
+                    sitemap_url, self._recursion_level, parent_urls
+                ):
+                    fetcher = SitemapFetcher(
+                        url=sitemap_url,
+                        recursion_level=self._recursion_level + 1,
+                        web_client=self._web_client,
+                        parent_urls=parent_urls,
+                        recurse_callback=self._recurse_callback,
+                        recurse_list_callback=self._recurse_list_callback,
+                    )
+                    fetched_sitemap = fetcher.sitemap()
+                else:
+                    continue
             except NoWebClientException:
                 fetched_sitemap = InvalidSitemap(
                     url=sitemap_url, reason="Un-fetched child sitemap"
@@ -376,6 +420,8 @@ class XMLSitemapParser(AbstractSitemapParser):
         recursion_level: int,
         web_client: AbstractWebClient,
         parent_urls: Set[str],
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
     ):
         super().__init__(
             url=url,
@@ -383,6 +429,8 @@ class XMLSitemapParser(AbstractSitemapParser):
             recursion_level=recursion_level,
             web_client=web_client,
             parent_urls=parent_urls,
+            recurse_callback=recurse_callback,
+            recurse_list_callback=recurse_list_callback,
         )
 
         # Will be initialized when the type of sitemap is known
@@ -491,6 +539,8 @@ class XMLSitemapParser(AbstractSitemapParser):
                     web_client=self._web_client,
                     recursion_level=self._recursion_level,
                     parent_urls=self._parent_urls,
+                    recurse_callback=self._recurse_callback,
+                    recurse_list_callback=self._recurse_list_callback,
                 )
 
             elif name == "rss":
@@ -536,13 +586,30 @@ class AbstractXMLSitemapParser(metaclass=abc.ABCMeta):
         # Last encountered character data
         "_last_char_data",
         "_last_handler_call_was_xml_char_data",
+        "_recurse_callback",
+        "_recurse_list_callback",
     ]
 
-    def __init__(self, url: str):
+    def __init__(
+        self,
+        url: str,
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
+    ):
         self._url = url
         self._last_char_data = ""
         self._last_handler_call_was_xml_char_data = False
 
+        if recurse_callback is None:  # Always allow child recursion
+            self._recurse_callback = lambda url, level, parent_urls: True
+        else:
+            self._recurse_callback = recurse_callback
+
+        if recurse_list_callback is None:  # Always allow child recursion
+            self._recurse_list_callback = lambda urls, level, parent_urls: urls
+        else:
+            self._recurse_list_callback = recurse_list_callback
+
     def xml_element_start(self, name: str, attrs: Dict[str, str]) -> None:
         """Concrete parser handler when the start of an element is encountered.
 
@@ -613,8 +680,14 @@ class IndexXMLSitemapParser(AbstractXMLSitemapParser):
         web_client: AbstractWebClient,
         recursion_level: int,
         parent_urls: Set[str],
+        recurse_callback: Optional[RecurseCallbackType] = None,
+        recurse_list_callback: Optional[RecurseListCallbackType] = None,
     ):
-        super().__init__(url=url)
+        super().__init__(
+            url=url,
+            recurse_callback=recurse_callback,
+            recurse_list_callback=recurse_list_callback,
+        )
 
         self._web_client = web_client
         self._recursion_level = recursion_level
@@ -638,16 +711,27 @@ class IndexXMLSitemapParser(AbstractXMLSitemapParser):
     def sitemap(self) -> AbstractSitemap:
         sub_sitemaps = []
 
-        for sub_sitemap_url in self._sub_sitemap_urls:
+        parent_urls = self._parent_urls | {self._url}
+        filtered_sitemap_urls = self._recurse_list_callback(
+            list(self._sub_sitemap_urls), self._recursion_level, parent_urls
+        )
+        for sub_sitemap_url in filtered_sitemap_urls:
             # URL might be invalid, or recursion limit might have been reached
             try:
-                fetcher = SitemapFetcher(
-                    url=sub_sitemap_url,
-                    recursion_level=self._recursion_level + 1,
-                    web_client=self._web_client,
-                    parent_urls=self._parent_urls | {self._url},
-                )
-                fetched_sitemap = fetcher.sitemap()
+                if self._recurse_callback(
+                    sub_sitemap_url, self._recursion_level, parent_urls
+                ):
+                    fetcher = SitemapFetcher(
+                        url=sub_sitemap_url,
+                        recursion_level=self._recursion_level + 1,
+                        web_client=self._web_client,
+                        parent_urls=parent_urls,
+                        recurse_callback=self._recurse_callback,
+                        recurse_list_callback=self._recurse_list_callback,
+                    )
+                    fetched_sitemap = fetcher.sitemap()
+                else:
+                    continue
             except NoWebClientException:
                 fetched_sitemap = InvalidSitemap(
                     url=sub_sitemap_url, reason="Un-fetched child sitemap"

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/usp/helpers.py

@@ -8,7 +8,7 @@ import re
 import sys
 import time
 from http import HTTPStatus
-from typing import Optional
+from typing import Callable, List, Optional, Set
 from urllib.parse import unquote_plus, urlparse, urlunparse
 
 from dateutil.parser import isoparse as dateutil_isoparse
@@ -29,6 +29,18 @@ __URL_REGEX = re.compile(r"^https?://[^\s/$.?#].[^\s]*$", re.IGNORECASE)
 
 HAS_DATETIME_NEW_ISOPARSER = sys.version_info >= (3, 11)
 
+# TODO: Convert to TypeAlias when Python3.9 support is dropped.
+RecurseCallbackType = Callable[[str, int, Set[str]], bool]
+"""Type for the callback function used to decide whether to recurse into a sitemap.
+
+A function that takes the sub-sitemap URL, the current recursion level, and the set of parent URLs as arguments, and returns a boolean indicating whether to recurse into the sub-sitemap.
+"""
+RecurseListCallbackType = Callable[[List[str], int, Set[str]], List[str]]
+"""Type for the callback function used to filter the list of sitemaps to recurse into.
+
+A function that takes the list of sub-sitemap URLs, the current recursion level, and the set of parent URLs as arguments, and returns a list of sub-sitemap URLs to recurse into.
+"""
+
 
 def is_http_url(url: str) -> bool:
     """

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/usp/objects/sitemap.py

@@ -230,7 +230,10 @@ class AbstractPagesSitemap(AbstractSitemap, metaclass=abc.ABCMeta):
             pickle.dump(pages, tmp, protocol=pickle.HIGHEST_PROTOCOL)
 
     def __del__(self):
-        os.unlink(self.__pages_temp_file_path)
+        try:
+            os.unlink(self.__pages_temp_file_path)
+        except FileNotFoundError as e:
+            log.warning("Unable to remove temp file", exc_info=e)
 
     def __eq__(self, other) -> bool:
         if not isinstance(other, AbstractPagesSitemap):

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/usp/tree.py

@@ -5,7 +5,12 @@ from typing import Optional
 
 from .exceptions import SitemapException
 from .fetch_parse import SitemapFetcher, SitemapStrParser
-from .helpers import is_http_url, strip_url_to_homepage
+from .helpers import (
+    RecurseCallbackType,
+    RecurseListCallbackType,
+    is_http_url,
+    strip_url_to_homepage,
+)
 from .objects.sitemap import (
     AbstractSitemap,
     IndexRobotsTxtSitemap,
@@ -41,6 +46,8 @@ def sitemap_tree_for_homepage(
     use_robots: bool = True,
     use_known_paths: bool = True,
     extra_known_paths: Optional[set] = None,
+    recurse_callback: Optional[RecurseCallbackType] = None,
+    recurse_list_callback: Optional[RecurseListCallbackType] = None,
 ) -> AbstractSitemap:
     """
     Using a homepage URL, fetch the tree of sitemaps and pages listed in them.
@@ -51,6 +58,8 @@ def sitemap_tree_for_homepage(
     :param use_robots: Whether to discover sitemaps through robots.txt.
     :param use_known_paths: Whether to discover sitemaps through common known paths.
     :param extra_known_paths: Extra paths to check for sitemaps.
+    :param recurse_callback: Optional callback function to determine if a sub-sitemap should be recursed into. See :data:`~.RecurseCallbackType`.
+    :param recurse_list_callback: Optional callback function to filter the list of sub-sitemaps to recurse into. See :data:`~.RecurseListCallbackType`.
     :return: Root sitemap object of the fetched sitemap tree.
     """
 
@@ -79,6 +88,8 @@ def sitemap_tree_for_homepage(
             web_client=web_client,
             recursion_level=0,
             parent_urls=set(),
+            recurse_callback=recurse_callback,
+            recurse_list_callback=recurse_list_callback,
         )
         robots_txt_sitemap = robots_txt_fetcher.sitemap()
         if not isinstance(robots_txt_sitemap, InvalidSitemap):
@@ -100,6 +111,8 @@ def sitemap_tree_for_homepage(
                     recursion_level=0,
                     parent_urls=sitemap_urls_found_in_robots_txt,
                     quiet_404=True,
+                    recurse_callback=recurse_callback,
+                    recurse_list_callback=recurse_list_callback,
                 )
                 unpublished_sitemap = unpublished_sitemap_fetcher.sitemap()
 

{ultimate_sitemap_parser-1.4.0 → ultimate_sitemap_parser-1.6.0}/usp/web_client/requests_client.py

@@ -79,7 +79,7 @@ class RequestsWebClient(AbstractWebClient):
 
     __USER_AGENT = f"ultimate_sitemap_parser/{__version__}"
 
-    __HTTP_REQUEST_TIMEOUT = 60
+    __HTTP_REQUEST_TIMEOUT = (9.05, 60)
     """
     HTTP request timeout.
 
@@ -114,7 +114,7 @@ class RequestsWebClient(AbstractWebClient):
         self.__waiter = RequestWaiter(wait, random_wait)
         self.__session = session or requests.Session()
 
-    def set_timeout(self, timeout: Union[int, Tuple[int, int], None]) -> None:
+    def set_timeout(self, timeout: Optional[Union[float, Tuple[float, float]]]) -> None:
         """Set HTTP request timeout.
 
         See also: `Requests timeout docs <https://requests.readthedocs.io/en/latest/user/advanced/#timeouts>`__