ultimate-sitemap-parser 1.0.0rc1__py3-none-any.whl

ultimate_sitemap_parser-1.0.0rc1.dist-info/METADATA

@@ -0,0 +1,109 @@
+Metadata-Version: 2.1
+Name: ultimate-sitemap-parser
+Version: 1.0.0rc1
+Summary: A performant library for parsing and crawling sitemaps
+Home-page: https://ultimate-sitemap-parser.readthedocs.io/
+License: GPL-3.0-or-later
+Keywords: sitemap,crawler,indexing,xml,rss,atom,google news
+Author: Linas Valiukas
+Author-email: linas@media.mit.edu
+Maintainer: Freddy Heppell
+Maintainer-email: f.heppell@sheffield.ac.uk
+Requires-Python: >=3.8,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Text Processing :: Markup :: XML
+Requires-Dist: python-dateutil (>=2.7,<3.0.0)
+Requires-Dist: requests (>=2.2.1)
+Project-URL: Documentation, https://ultimate-sitemap-parser.readthedocs.io/
+Project-URL: Repository, https://github.com/GateNLP/ultimate-sitemap-parser
+Description-Content-Type: text/x-rst
+
+Ultimate Sitemap Parser
+-----------------------
+
+.. image:: https://img.shields.io/pypi/pyversions/ultimate-sitemap-parser
+   :alt: PyPI - Python Version
+   :target: https://github.com/GateNLP/ultimate-sitemap-parser
+
+.. image:: https://img.shields.io/pypi/v/ultimate-sitemap-parser
+   :alt: PyPI - Version
+   :target: https://pypi.org/project/ultimate-sitemap-parser/
+
+.. image:: https://img.shields.io/conda/vn/conda-forge/ultimate-sitemap-parser
+   :alt: Conda Version
+   :target: https://anaconda.org/conda-forge/ultimate-sitemap-parser
+
+.. image:: https://img.shields.io/pepy/dt/ultimate-sitemap-parser
+   :target: https://pepy.tech/project/ultimate-sitemap-parser
+   :alt: Pepy Total Downloads
+
+
+**Ultimate Sitemap Parser (USP) is a performant and robust Python library for parsing and crawling sitemaps.**
+
+
+Features
+========
+
+- Supports all sitemap formats:
+
+  - `XML sitemaps <https://www.sitemaps.org/protocol.html#xmlTagDefinitions>`_
+  - `Google News sitemaps <https://developers.google.com/search/docs/crawling-indexing/sitemaps/news-sitemap>`_ and `Image sitemaps <https://developers.google.com/search/docs/advanced/sitemaps/image-sitemaps>`_
+  - `plain text sitemaps <https://www.sitemaps.org/protocol.html#otherformats>`_
+  - `RSS 2.0 / Atom 0.3 / Atom 1.0 sitemaps <https://www.sitemaps.org/protocol.html#otherformats>`_
+  - `Sitemaps linked from robots.txt <https://developers.google.com/search/reference/robots_txt#sitemap>`_
+
+- Field-tested with ~1 million URLs as part of the `Media Cloud project <https://mediacloud.org/>`_
+- Error-tolerant with more common sitemap bugs
+- Tries to find sitemaps not listed in ``robots.txt``
+- Uses fast and memory efficient Expat XML parsing
+- Doesn't consume much memory even with massive sitemap hierarchies
+- Provides a generated sitemap tree as easy to use object tree
+- Supports using a custom web client
+- Uses a small number of actively maintained third-party modules
+- Reasonably tested
+
+
+Installation
+============
+
+.. code:: sh
+
+    pip install ultimate-sitemap-parser
+
+or using Anaconda:
+
+.. code:: sh
+
+    conda install -c conda-forge ultimate-sitemap-parser
+
+
+Usage
+=====
+
+.. code:: python
+
+    from usp.tree import sitemap_tree_for_homepage
+
+    tree = sitemap_tree_for_homepage('https://www.example.org/')
+
+    for page in tree.all_pages():
+        print(page.url)
+
+``sitemap_tree_for_homepage()`` will return a tree of ``AbstractSitemap`` subclass objects that represent the sitemap
+hierarchy found on the website; see a `reference of AbstractSitemap subclasses <https://ultimate-sitemap-parser.readthedocs.io/en/latest/reference/api/usp.objects.sitemap.html>`_. `AbstractSitemap.all_pages()` returns a generator to efficiently iterate over pages without loading the entire tree into memory.
+
+For more examples and details, see the `documentation <https://ultimate-sitemap-parser.readthedocs.io/en/latest/>`_.
+

ultimate_sitemap_parser-1.0.0rc1.dist-info/NOTICE

@@ -0,0 +1,12 @@
+Copyright (C) 2018 Linas Valiukas, Hal Roberts, Media Cloud project,
+    Freddy Heppell, The University of Sheffield, and other contributors.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details <http://www.gnu.org/licenses/>.

ultimate_sitemap_parser-1.0.0rc1.dist-info/RECORD

@@ -0,0 +1,22 @@
+usp/__init__.py,sha256=_jshbOBBUHRZ5ko4SdI7GRFiF9xKGJVCEPgL9lZJ81o,124
+usp/cli/__init__.py,sha256=mGrjSftUYfM2SGp9yEN2dTJndl5thOdv77-EAe6ocWo,37
+usp/cli/_ls.py,sha256=YyDmtBjK02_26Qv8-3NLf87b1C4Wt0GzZ1XkdF2fllQ,2954
+usp/cli/_util.py,sha256=UL5WiRZlpiDOI_QvSU1PdjcS6iCmfcLQlO1Mm1wjSAw,505
+usp/cli/cli.py,sha256=D1tXZyhiG0sIwtepdPdglW5gUlPWyx4LNeBmaM700Yc,592
+usp/exceptions.py,sha256=9KTgnocYYZCfyaCf9BrBN7Ok4cwn7_DlrNFbhUfFsGM,634
+usp/fetch_parse.py,sha256=aE0ohttrce_gcZE8iBFvjy6iRJEEij0SzIOX3PEsDXw,40571
+usp/helpers.py,sha256=QJH3ETapqqbwRnjX_LM0EhWqeta9LTqVvW5OAkBKUOc,8491
+usp/log.py,sha256=BS0AtURK62TPGVqEuIu8kwGtIJDYoGsK5_N-b60VOpE,1631
+usp/objects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+usp/objects/page.py,sha256=PNuYGburGu6AqoIdI8NdNd_6XG7nyyG8qlXKekuIinY,13483
+usp/objects/sitemap.py,sha256=XWgke1SJNA79qnOEvaY2nJbnlidWxqBvfuRcF4GhBHI,11564
+usp/tree.py,sha256=2cuHOpdYX5aKZ4XuUQPaKjILnoPnFKZwPNn0g8cxT18,4066
+usp/web_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+usp/web_client/abstract_client.py,sha256=4OdmM9UEjK3LUYZCRwYVDrmapKkPXHR7fYch47W5xys,5381
+usp/web_client/requests_client.py,sha256=N2nY15Aj9wd-qF0ujS4-YvMWaRntfWnX8ze5I_h7FnI,5079
+ultimate_sitemap_parser-1.0.0rc1.dist-info/LICENSE,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
+ultimate_sitemap_parser-1.0.0rc1.dist-info/METADATA,sha256=af_IqHD8nO8DzsiiG17faK52sMh6Br0imarNxZthhmY,4385
+ultimate_sitemap_parser-1.0.0rc1.dist-info/NOTICE,sha256=3ANZA5R9rYnCOnUoroGfFUOZ__ww_yG01NUAx0X6J7E,632
+ultimate_sitemap_parser-1.0.0rc1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+ultimate_sitemap_parser-1.0.0rc1.dist-info/entry_points.txt,sha256=v60w5WzqYlPOucntZUy0ydzlYwuAPSwoQY0KdT5ragQ,36
+ultimate_sitemap_parser-1.0.0rc1.dist-info/RECORD,,

ultimate_sitemap_parser-1.0.0rc1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

ultimate_sitemap_parser-1.0.0rc1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+usp=usp.cli:main
+

usp/__init__.py

@@ -0,0 +1,5 @@
+from importlib.metadata import version
+
+__version__ = version("ultimate-sitemap-parser")
+
+__all__ = ["tree", "__version__"]

usp/cli/__init__.py

@@ -0,0 +1 @@
+from usp.cli.cli import main as main

usp/cli/_ls.py

@@ -0,0 +1,105 @@
+import argparse
+import sys
+from typing import Iterator
+
+from usp.cli._util import tabs, format_help
+from usp.objects.sitemap import AbstractSitemap
+from usp.tree import sitemap_tree_for_homepage
+
+LS_FORMATS = {
+    "tabtree": "Sitemaps and pages, nested with tab indentation",
+    "pages": "Flat list of pages, one per line",
+}
+
+
+def register(subparsers):
+    ls_parser = subparsers.add_parser(
+        "ls",
+        help="List sitemap pages",
+        description="download, parse and list the sitemap structure",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    ls_parser.add_argument("url", type=str, help="URL of the site including protocol")
+    ls_parser.add_argument(
+        "-f",
+        "--format",
+        choices=LS_FORMATS,
+        default="tabtree",
+        help=format_help(LS_FORMATS, "set output format"),
+        metavar="",
+    )
+    ls_parser.add_argument(
+        "-r",
+        "--no-robots",
+        action="store_true",
+        help="don't discover sitemaps through robots.txt",
+    )
+    ls_parser.add_argument(
+        "-k",
+        "--no-known",
+        action="store_true",
+        help="don't discover sitemaps through well-known URLs",
+    )
+    ls_parser.add_argument(
+        "-u",
+        "--strip-url",
+        action="store_true",
+        help="strip the supplied URL from each page and sitemap URL",
+    )
+    ls_parser.set_defaults(no_robots=False, no_known=False, strip_url=False)
+
+    ls_parser.set_defaults(func=ls)
+
+
+def _strip_url(url: str, prefix: str):
+    url = url.removeprefix(prefix)
+
+    if not url.startswith("/") and prefix != "":
+        return "/" + url
+    return url
+
+
+def _list_page_urls(sitemap: AbstractSitemap, prefix: str = "") -> Iterator[str]:
+    for page in sitemap.all_pages():
+        yield prefix + page.url
+
+
+def _output_sitemap_nested(
+    sitemap: AbstractSitemap, strip_prefix: str = "", depth: int = 0
+):
+    sitemap_url = sitemap.url
+    if depth != 0:
+        sitemap_url = _strip_url(sitemap_url, strip_prefix)
+    sys.stdout.write(tabs(depth) + sitemap_url + "\n")
+
+    for sub_map in sitemap.sub_sitemaps:
+        _output_sitemap_nested(sub_map, strip_prefix, depth + 1)
+
+    for page in sitemap.pages:
+        sys.stdout.write(tabs(depth + 1) + _strip_url(page.url, strip_prefix) + "\n")
+
+
+def _output_pages(sitemap: AbstractSitemap, strip_prefix: str = ""):
+    for page in sitemap.all_pages():
+        sys.stdout.write(_strip_url(page.url, strip_prefix) + "\n")
+
+
+def ls(args):
+    tree = sitemap_tree_for_homepage(
+        args.url,
+        use_robots=not args.no_robots,
+        use_known_paths=not args.no_known,
+    )
+
+    strip_prefix = ""
+    if args.strip_url:
+        strip_prefix = tree.url
+
+    if args.format == "pages":
+        _output_pages(tree, strip_prefix)
+    elif args.format == "tabtree":
+        _output_sitemap_nested(tree, strip_prefix)
+    else:
+        raise NotImplementedError(f"Format '{args.format}' not implemented")
+
+    exit(0)

usp/cli/_util.py

@@ -0,0 +1,21 @@
+from typing import Dict
+
+
+def format_help(choices: Dict[str, str], opt_help: str) -> str:
+    """Generate help text for argparse choices.
+
+    :param choices: Dictionary of choices {choice: help}
+    :param opt_help: Help text for the option:
+    :return: Help text for argparse choices.
+    """
+    h = f"{opt_help} (default: %(default)s)\nchoices:\n"
+
+    for fmt, key in choices.items():
+        h += f"  {fmt}: {key}\n"
+
+    return h
+
+
+def tabs(n: int):
+    """Generate n tabs."""
+    return "\t" * n

usp/cli/cli.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+
+from usp.cli import _ls as ls_cmd
+from usp import __version__
+
+
+def main():
+    parser = ArgumentParser(prog="usp", description="Ultimate Sitemap Parser")
+    parser.add_argument(
+        "-v", "--version", action="version", version=f"%(prog)s v{__version__}"
+    )
+
+    subparsers = parser.add_subparsers(required=False, title="commands", metavar="")
+    ls_cmd.register(subparsers)
+
+    args = parser.parse_args()
+
+    if "func" in args:
+        args.func(args)
+    else:
+        parser.print_help()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

usp/exceptions.py

@@ -0,0 +1,35 @@
+"""Exceptions used by the sitemap parser."""
+
+
+class SitemapException(Exception):
+    """
+    Problem due to which we can't run further, e.g. wrong input parameters.
+    """
+
+    pass
+
+
+class SitemapXMLParsingException(Exception):
+    """
+    XML parsing exception to be handled gracefully.
+    """
+
+    pass
+
+
+class GunzipException(Exception):
+    """
+    Error decompressing seemingly gzipped content.
+    See :func:`usp.helpers.gunzip`.
+    """
+
+    pass
+
+
+class StripURLToHomepageException(Exception):
+    """
+    Problem parsing URL and stripping to homepage.
+    See :func:`usp.helpers.strip_url_to_homepage`.
+    """
+
+    pass