ultimate-sitemap-parser 1.3.1__py3-none-any.whl → 1.4.0__py3-none-any.whl

{ultimate_sitemap_parser-1.3.1.dist-info → ultimate_sitemap_parser-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ultimate-sitemap-parser
-Version: 1.3.1
+Version: 1.4.0
 Summary: A performant library for parsing and crawling sitemaps
 License: GPL-3.0-or-later
 Keywords: sitemap,crawler,indexing,xml,rss,atom,google news

{ultimate_sitemap_parser-1.3.1.dist-info → ultimate_sitemap_parser-1.4.0.dist-info}/RECORD

@@ -4,8 +4,8 @@ usp/cli/_ls.py,sha256=V0pMsDiQK_9RZ5MyUS2toW8b6e2FJ4spb3Grw6PayAI,3419
 usp/cli/_util.py,sha256=OrT9en350tATnaUrUn0peXr7aFPyYaaHGbEXGY6O4wI,2015
 usp/cli/cli.py,sha256=2byuqhBUhb7c1qUpBfTTufG-jvtiEWWq97GvCgv-s44,777
 usp/exceptions.py,sha256=9KTgnocYYZCfyaCf9BrBN7Ok4cwn7_DlrNFbhUfFsGM,634
-usp/fetch_parse.py,sha256=69U1uAKawUym41N4nwJXLW9tQ0WXO4Pi63hnljYCXPM,43524
-usp/helpers.py,sha256=FeIZcEuEM3Uz8tHeNucgoB3_27Ax6qCatfalPIHHGUY,8862
+usp/fetch_parse.py,sha256=COngyIf0ifmv_XAegUKzEn74pwdak9F6Thi7DSbglRs,44312
+usp/helpers.py,sha256=OSP5W9N1WFc7MDZxYnGLvHKLodkkKUrq2ND65h9l-9Q,8726
 usp/objects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 usp/objects/page.py,sha256=vz2QXC9Z3E65Cxf68tKfQkubIc_OB0m6pNYH146Qx_8,14253
 usp/objects/sitemap.py,sha256=_t0ej2UmNsIb0NkxYkwYGxBqX_LHEJfNc-cRulQXyIk,11495
@@ -13,9 +13,9 @@ usp/tree.py,sha256=MdnVxfIIMqWrudsYxFI8yQTXnlmNLFEcQEOkXbnuBr4,4395
 usp/web_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 usp/web_client/abstract_client.py,sha256=EWY4lPYJqpV7ge0DTZESTAOofAjNMIJnDm_2PPeZ9z4,7007
 usp/web_client/requests_client.py,sha256=sFYtJ8Q5z27WlTG1PgBzcvbS75pJ0pYUastEFmxa95U,5888
-ultimate_sitemap_parser-1.3.1.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
-ultimate_sitemap_parser-1.3.1.dist-info/METADATA,sha256=GUU8qLo24ZGBtAd4CYaHxY927eFFGvKlVPhc6jfg5so,4397
-ultimate_sitemap_parser-1.3.1.dist-info/NOTICE,sha256=3ANZA5R9rYnCOnUoroGfFUOZ__ww_yG01NUAx0X6J7E,632
-ultimate_sitemap_parser-1.3.1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-ultimate_sitemap_parser-1.3.1.dist-info/entry_points.txt,sha256=v60w5WzqYlPOucntZUy0ydzlYwuAPSwoQY0KdT5ragQ,36
-ultimate_sitemap_parser-1.3.1.dist-info/RECORD,,
+ultimate_sitemap_parser-1.4.0.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
+ultimate_sitemap_parser-1.4.0.dist-info/METADATA,sha256=L2nASfrjc5-hZL71JAmf6h6SUeeAVHyppi5fbuDt5lo,4397
+ultimate_sitemap_parser-1.4.0.dist-info/NOTICE,sha256=3ANZA5R9rYnCOnUoroGfFUOZ__ww_yG01NUAx0X6J7E,632
+ultimate_sitemap_parser-1.4.0.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+ultimate_sitemap_parser-1.4.0.dist-info/entry_points.txt,sha256=v60w5WzqYlPOucntZUy0ydzlYwuAPSwoQY0KdT5ragQ,36
+ultimate_sitemap_parser-1.4.0.dist-info/RECORD,,

usp/fetch_parse.py

@@ -366,6 +366,7 @@ class XMLSitemapParser(AbstractSitemapParser):
 
     __slots__ = [
         "_concrete_parser",
+        "_is_non_ns_sitemap",
     ]
 
     def __init__(
@@ -386,6 +387,8 @@ class XMLSitemapParser(AbstractSitemapParser):
 
         # Will be initialized when the type of sitemap is known
         self._concrete_parser = None
+        # Whether this is a malformed sitemap with no namespace
+        self._is_non_ns_sitemap = False
 
     def sitemap(self) -> AbstractSitemap:
         parser = xml.parsers.expat.ParserCreate(
@@ -411,8 +414,7 @@ class XMLSitemapParser(AbstractSitemapParser):
 
         return self._concrete_parser.sitemap()
 
-    @classmethod
-    def __normalize_xml_element_name(cls, name: str):
+    def __normalize_xml_element_name(self, name: str):
         """
         Replace the namespace URL in the argument element name with internal namespace.
 
@@ -428,7 +430,7 @@ class XMLSitemapParser(AbstractSitemapParser):
         :return: Internal namespace name plus element name, e.g. "sitemap loc"
         """
 
-        name_parts = name.split(cls.__XML_NAMESPACE_SEPARATOR)
+        name_parts = name.split(self.__XML_NAMESPACE_SEPARATOR)
 
         if len(name_parts) == 1:
             namespace_url = ""
@@ -451,6 +453,19 @@ class XMLSitemapParser(AbstractSitemapParser):
             name = f"image:{name}"
         elif "/sitemap-video/" in namespace_url:
             name = f"video:{name}"
+        elif name in {"urlset", "sitemapindex"}:
+            # XML sitemap root tag but namespace is not set
+            self._is_non_ns_sitemap = True
+            log.warning(
+                f'XML sitemap root tag {name} detected without expected xmlns (value is "{namespace_url}"), '
+                f"assuming is an XML sitemap."
+            )
+            name = f"sitemap:{name}"
+        elif self._is_non_ns_sitemap:
+            # Flag has previously been set and no other namespace matched,
+            # assume this should be in the sitemap namespace
+            log.debug(f"Assuming {name} should be in sitemap namespace")
+            name = f"sitemap:{name}"
         else:
             # We don't care about the rest of the namespaces, so just keep the plain element name
             pass

usp/helpers.py

@@ -194,13 +194,8 @@ def __response_is_gzipped_data(
     uri = urlparse(url)
     url_path = unquote_plus(uri.path)
     content_type = response.header("content-type") or ""
-    content_encoding = response.header("content-encoding") or ""
 
-    if (
-        url_path.lower().endswith(".gz")
-        or "gzip" in content_type.lower()
-        or "gzip" in content_encoding.lower()
-    ):
+    if url_path.lower().endswith(".gz") or "gzip" in content_type.lower():
         return True
 
     else:
@@ -260,7 +255,7 @@ def ungzipped_response_content(
         except GunzipException as ex:
             # In case of an error, just assume that it's one of the non-gzipped sitemaps with ".gz" extension
             log.warning(
-                f"Unable to gunzip response {response}, maybe it's a non-gzipped sitemap: {ex}"
+                f"Unable to gunzip response for {url}, maybe it's a non-gzipped sitemap: {ex}"
             )
 
     # FIXME other encodings