ultimate-sitemap-parser 1.3.1__tar.gz → 1.4.0__tar.gz

{ultimate_sitemap_parser-1.3.1 → ultimate_sitemap_parser-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ultimate-sitemap-parser
-Version: 1.3.1
+Version: 1.4.0
 Summary: A performant library for parsing and crawling sitemaps
 License: GPL-3.0-or-later
 Keywords: sitemap,crawler,indexing,xml,rss,atom,google news

{ultimate_sitemap_parser-1.3.1 → ultimate_sitemap_parser-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ultimate-sitemap-parser"
-version = "1.3.1"
+version = "1.4.0"
 description = "A performant library for parsing and crawling sitemaps"
 authors = [
     { name = "Linas Valiukas", email = "linas@media.mit.edu"},
@@ -53,7 +53,7 @@ python = ">=3.9,<4.0"
 [tool.poetry.group.dev.dependencies]
 requests-mock = ">=1.6.0,<2.0"
 pytest = "^8.3.0"
-ruff = "^0.9.3"
+ruff = "^0.11.6"
 vcrpy = "6.0.1"
 pytest-mock = "^3.14.0"
 

{ultimate_sitemap_parser-1.3.1 → ultimate_sitemap_parser-1.4.0}/usp/fetch_parse.py

@@ -366,6 +366,7 @@ class XMLSitemapParser(AbstractSitemapParser):
 
     __slots__ = [
         "_concrete_parser",
+        "_is_non_ns_sitemap",
     ]
 
     def __init__(
@@ -386,6 +387,8 @@ class XMLSitemapParser(AbstractSitemapParser):
 
         # Will be initialized when the type of sitemap is known
         self._concrete_parser = None
+        # Whether this is a malformed sitemap with no namespace
+        self._is_non_ns_sitemap = False
 
     def sitemap(self) -> AbstractSitemap:
         parser = xml.parsers.expat.ParserCreate(
@@ -411,8 +414,7 @@ class XMLSitemapParser(AbstractSitemapParser):
 
         return self._concrete_parser.sitemap()
 
-    @classmethod
-    def __normalize_xml_element_name(cls, name: str):
+    def __normalize_xml_element_name(self, name: str):
         """
         Replace the namespace URL in the argument element name with internal namespace.
 
@@ -428,7 +430,7 @@ class XMLSitemapParser(AbstractSitemapParser):
         :return: Internal namespace name plus element name, e.g. "sitemap loc"
         """
 
-        name_parts = name.split(cls.__XML_NAMESPACE_SEPARATOR)
+        name_parts = name.split(self.__XML_NAMESPACE_SEPARATOR)
 
         if len(name_parts) == 1:
             namespace_url = ""
@@ -451,6 +453,19 @@ class XMLSitemapParser(AbstractSitemapParser):
             name = f"image:{name}"
         elif "/sitemap-video/" in namespace_url:
             name = f"video:{name}"
+        elif name in {"urlset", "sitemapindex"}:
+            # XML sitemap root tag but namespace is not set
+            self._is_non_ns_sitemap = True
+            log.warning(
+                f'XML sitemap root tag {name} detected without expected xmlns (value is "{namespace_url}"), '
+                f"assuming is an XML sitemap."
+            )
+            name = f"sitemap:{name}"
+        elif self._is_non_ns_sitemap:
+            # Flag has previously been set and no other namespace matched,
+            # assume this should be in the sitemap namespace
+            log.debug(f"Assuming {name} should be in sitemap namespace")
+            name = f"sitemap:{name}"
         else:
             # We don't care about the rest of the namespaces, so just keep the plain element name
             pass

{ultimate_sitemap_parser-1.3.1 → ultimate_sitemap_parser-1.4.0}/usp/helpers.py

@@ -194,13 +194,8 @@ def __response_is_gzipped_data(
     uri = urlparse(url)
     url_path = unquote_plus(uri.path)
     content_type = response.header("content-type") or ""
-    content_encoding = response.header("content-encoding") or ""
 
-    if (
-        url_path.lower().endswith(".gz")
-        or "gzip" in content_type.lower()
-        or "gzip" in content_encoding.lower()
-    ):
+    if url_path.lower().endswith(".gz") or "gzip" in content_type.lower():
         return True
 
     else:
@@ -260,7 +255,7 @@ def ungzipped_response_content(
         except GunzipException as ex:
             # In case of an error, just assume that it's one of the non-gzipped sitemaps with ".gz" extension
             log.warning(
-                f"Unable to gunzip response {response}, maybe it's a non-gzipped sitemap: {ex}"
+                f"Unable to gunzip response for {url}, maybe it's a non-gzipped sitemap: {ex}"
             )
 
     # FIXME other encodings