scrapyloganalyzer 0.0.1__tar.gz

scrapyloganalyzer-0.0.1/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Open Contracting Partnership
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

scrapyloganalyzer-0.0.1/MANIFEST.in

@@ -0,0 +1,9 @@
+include LICENSE
+recursive-include docs *.py
+recursive-include docs *.rst
+recursive-include docs *.txt
+recursive-include docs Makefile
+recursive-include tests *.log
+recursive-include tests *.py
+exclude .pre-commit-config.yaml
+exclude .readthedocs.yaml

scrapyloganalyzer-0.0.1/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: scrapyloganalyzer
+Version: 0.0.1
+Summary: Provides methods to analyze the quality of a Scrapy crawl.
+Author-email: Open Contracting Partnership <data@open-contracting.org>
+License: BSD
+Project-URL: Homepage, https://github.com/open-contracting/scrapy-log-analyzer
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: logparser
+Provides-Extra: test
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pytest; extra == "test"
+
+|PyPI Version| |Build Status| |Coverage Status| |Python Version|
+
+Provides methods to analyze the quality of a Scrapy crawl.
+
+(If you are viewing this on GitHub, open the `full documentation <https://scrapy-log-analyzer.readthedocs.io/>`__ for additional details.)
+
+.. |PyPI Version| image:: https://img.shields.io/pypi/v/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/
+.. |Build Status| image:: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml/badge.svg
+   :target: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml
+.. |Coverage Status| image:: https://coveralls.io/repos/github/open-contracting/scrapy-log-analyzer/badge.svg?branch=main
+   :target: https://coveralls.io/github/open-contracting/scrapy-log-analyzer?branch=main
+.. |Python Version| image:: https://img.shields.io/pypi/pyversions/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/

scrapyloganalyzer-0.0.1/README.rst

@@ -0,0 +1,14 @@
+|PyPI Version| |Build Status| |Coverage Status| |Python Version|
+
+Provides methods to analyze the quality of a Scrapy crawl.
+
+(If you are viewing this on GitHub, open the `full documentation <https://scrapy-log-analyzer.readthedocs.io/>`__ for additional details.)
+
+.. |PyPI Version| image:: https://img.shields.io/pypi/v/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/
+.. |Build Status| image:: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml/badge.svg
+   :target: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml
+.. |Coverage Status| image:: https://coveralls.io/repos/github/open-contracting/scrapy-log-analyzer/badge.svg?branch=main
+   :target: https://coveralls.io/github/open-contracting/scrapy-log-analyzer?branch=main
+.. |Python Version| image:: https://img.shields.io/pypi/pyversions/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/

scrapyloganalyzer-0.0.1/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

scrapyloganalyzer-0.0.1/docs/api/index.rst

@@ -0,0 +1,6 @@
+API reference
+=============
+
+.. automodule:: scrapyloganalyzer
+   :members:
+   :undoc-members:

scrapyloganalyzer-0.0.1/docs/changelog.rst

@@ -0,0 +1,7 @@
+Changelog
+=========
+
+0.0.1 (2024-12-13)
+------------------
+
+First release.

scrapyloganalyzer-0.0.1/docs/conf.py

@@ -0,0 +1,75 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(".."))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "Scrapy Log Analyzer"
+copyright = "2022, Open Contracting Partnership"
+author = "Open Contracting Partnership"
+
+# The short X.Y version
+version = "0.0.1"
+# The full version, including alpha/beta/rc tags
+release = version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.viewcode",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "furo"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+
+# -- Extension configuration -------------------------------------------------
+
+autodoc_default_options = {
+    "members": None,
+    "member-order": "bysource",
+}
+autodoc_typehints = "description"
+autodoc_type_aliases = {}
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+}

scrapyloganalyzer-0.0.1/docs/contributing/index.rst

@@ -0,0 +1,12 @@
+Contributing
+============
+
+Setup
+-----
+
+Set up the git pre-commit hook:
+
+.. code-block:: bash
+
+   pip install pre-commit
+   pre-commit install

scrapyloganalyzer-0.0.1/docs/index.rst

@@ -0,0 +1,13 @@
+Scrapy Log Analyzer |release|
+=============================
+
+.. include:: ../README.rst
+
+.. toctree::
+   :caption: Contents
+
+   api/index
+   contributing/index
+   changelog
+
+Copyright (c) 2022 Open Contracting Partnership, released under the BSD license

scrapyloganalyzer-0.0.1/docs/requirements.txt

@@ -0,0 +1 @@
+furo

scrapyloganalyzer-0.0.1/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["setuptools>=61.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "scrapyloganalyzer"
+version = "0.0.1"
+authors = [{name = "Open Contracting Partnership", email = "data@open-contracting.org"}]
+description = "Provides methods to analyze the quality of a Scrapy crawl."
+readme = "README.rst"
+license = {text = "BSD"}
+urls = {Homepage = "https://github.com/open-contracting/scrapy-log-analyzer"}
+classifiers = [
+    "License :: OSI Approved :: BSD License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = [
+    "logparser",
+]
+
+[project.optional-dependencies]
+test = [
+    "coverage",
+    "pytest",
+]
+
+[tool.setuptools.packages.find]
+exclude = [
+    "tests",
+    "tests.*",
+]
+
+[tool.ruff]
+line-length = 119
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "ANN", "C901", "COM812", "D203", "D212", "D415", "EM", "ISC001", "PERF203", "PLR091", "Q000",
+    "D104", "D205",
+    "DTZ",
+    "PTH",
+]
+
+[tool.ruff.lint.flake8-builtins]
+builtins-ignorelist = ["copyright"]
+
+[tool.ruff.lint.flake8-unused-arguments]
+ignore-variadic-names = true
+
+[tool.ruff.lint.per-file-ignores]
+"docs/conf.py" = ["D100", "INP001"]
+"tests/*" = [
+    "ARG001", "D", "FBT003", "INP001", "PLR2004", "S", "TRY003",
+]

scrapyloganalyzer-0.0.1/scrapyloganalyzer/__init__.py

@@ -0,0 +1,181 @@
+import ast
+import datetime
+import os
+import re
+from collections import defaultdict
+
+from logparser import parse
+from logparser.common import DATETIME_PATTERN, Common
+
+# Kingfisher Collect logs an INFO message starting with "Spider arguments:".
+SPIDER_ARGUMENTS_SEARCH_STRING = " INFO: Spider arguments: "
+
+MAXIMUM_TIMEDELTA = 3
+
+# Hotfix: https://github.com/my8100/logparser/pull/19
+Common.SIGTERM_PATTERN = re.compile(r"^%s[ ].+?:[ ](Received[ ]SIG(?:BREAK|INT|TERM)([ ]twice)?)," % DATETIME_PATTERN)  # noqa: UP031
+
+
+class ScrapyLogFile:
+    """A representation of a Scrapy log file."""
+
+    @classmethod
+    def find(cls, logs_directory, source_id, data_version):
+        """
+        Find and return the first matching log file for the given crawl.
+
+        :param str logs_directory: Kingfisher Collect's project directory within Scrapyd's logs_dir directory
+        :param str source_id: the spider's name
+        :param datetime.datetime data_version: the crawl directory's name, parsed as a datetime
+        """
+        source_directory = os.path.join(logs_directory, source_id)
+        if os.path.isdir(source_directory):
+            with os.scandir(source_directory) as it:
+                for entry in it:
+                    if entry.name.endswith(".log"):
+                        scrapy_log_file = ScrapyLogFile(entry.path)
+                        if scrapy_log_file.match(data_version):
+                            return scrapy_log_file
+        return None
+
+    def __init__(self, name) -> None:
+        """:param str name: the full path to the log file"""
+        self.name = name
+
+        self._logparser = None
+        self._item_counts = None
+        self._spider_arguments = None
+
+    def delete(self):
+        """Delete the log file and any log summary ending in ``.stats``."""
+        if os.path.isfile(self.name):
+            os.remove(self.name)
+        summary = f"{self.name}.stats"
+        if os.path.isfile(summary):
+            os.remove(summary)
+
+    # Logparser processing
+
+    @property
+    def logparser(self) -> dict:
+        """Return the output of `logparser <https://pypi.org/project/logparser/>`__."""
+        if self._logparser is None:
+            with open(self.name) as f:
+                # `taillines=0` sets the 'tail' key to all lines, so we set it to 1.
+                self._logparser = parse(f.read(), headlines=0, taillines=1)
+
+        return self._logparser
+
+    def match(self, data_version) -> bool:
+        """
+        Return whether the crawl directory's name, parsed as a datetime, is less than 3 seconds after the log file's
+        start time.
+        """
+        return 0 <= data_version.timestamp() - self.crawl_time.timestamp() < MAXIMUM_TIMEDELTA
+
+    @property
+    def crawl_time(self) -> datetime.datetime:
+        """
+        Return the ``crawl_time`` spider argument if set, or the ``start_time`` crawl statistic otherwise. If neither
+        is logged, return the time of the first log message.
+        """
+        crawl_time = self.spider_arguments.get("crawl_time")
+        if crawl_time:
+            return datetime.datetime.strptime(crawl_time, "%Y-%m-%dT%H:%M:%S")
+        if "start_time" in self.logparser["crawler_stats"]:
+            return eval(self.logparser["crawler_stats"]["start_time"]).replace(microsecond=0)  # noqa: S307
+        return datetime.datetime.fromtimestamp(self.logparser["first_log_timestamp"])
+
+    def is_finished(self) -> bool:
+        """
+        Return whether the log file contains a "Spider closed (finished)" log message or a ``finish_reason`` crawl
+        statistic set to "finished".
+        """
+        # See https://kingfisher-collect.readthedocs.io/en/latest/logs.html#check-the-reason-for-closing-the-spider
+        # logparser's `finish_reason` is "N/A" for an unclean shutdown, because crawl statistics aren't logged.
+        return self.logparser["finish_reason"] == "finished"
+
+    # Line-by-line processing
+
+    @property
+    def item_counts(self) -> dict:
+        """Return the number of each type of item, according to the log file."""
+        if self._item_counts is None:
+            self._process_line_by_line()
+
+        return self._item_counts
+
+    @property
+    def spider_arguments(self) -> dict:
+        """Return the spider's arguments."""
+        if self._spider_arguments is None:
+            self._process_line_by_line()
+
+        return self._spider_arguments
+
+    def is_complete(self) -> bool:
+        """Return whether the crawl collected a subset of the dataset, according to the log file."""
+        # See https://kingfisher-collect.readthedocs.io/en/latest/spiders.html#spider-arguments
+        return not any(
+            self.spider_arguments.get(arg)
+            for arg in (
+                "from_date",
+                "until_date",
+                "portal",
+                "publisher",
+                "system",
+                "sample",
+                "path",
+                "qs:",
+            )
+        )
+
+    def _process_line_by_line(self) -> None:
+        self._item_counts = defaultdict(int)
+        self._spider_arguments = {}
+
+        buf = []
+        with open(self.name) as f:
+            for line in f:
+                if buf or line.startswith("{"):
+                    buf.append(line.rstrip())
+                if buf and buf[-1].endswith("}"):
+                    try:
+                        # Scrapy logs items as dicts. FileError items, representing retrieval errors, are identified by
+                        # an 'errors' key. FileError items use only simple types, so `ast.literal_eval` can be used.
+                        item = ast.literal_eval("".join(buf))
+                        if "errors" in item:
+                            self._item_counts["FileError"] += 1
+                        elif "number" in item:
+                            self._item_counts["FileItem"] += 1
+                        elif "data_type" in item:
+                            self._item_counts["File"] += 1
+                    except ValueError:
+                        # Scrapy dumps stats as a dict, which uses `datetime.datetime` types that can't be parsed with
+                        # `ast.literal_eval`.
+                        pass
+                    buf = []
+
+                index = line.find(SPIDER_ARGUMENTS_SEARCH_STRING)
+                if index > -1:
+                    # `eval` is used, because the string can contain `datetime.date` and is written by trusted code in
+                    # Kingfisher Collect. Otherwise, we can modify the string so that `ast.literal_eval` can be used.
+                    self._spider_arguments = eval(line[index + len(SPIDER_ARGUMENTS_SEARCH_STRING) :])  # noqa: S307
+
+    # Mixed processing
+
+    @property
+    def error_rate(self) -> float:
+        """
+        Return an estimated lower bound of the true error rate.
+
+        Kingfisher Collect is expected to yield at most one FileError item per request leading to a File item, so the
+        true error rate can only be less than this estimated lower bound if Kingfisher Collect breaks this expectation.
+        On the other hand, the true error rate can easily be higher than the estimated lower bound; for example:
+
+        -  If the spider crawls 10 URLs, each returning 99 URLs, each returning OCDS data, and the requests for 5 of
+           the 10 fail, then the estimated lower bound is 5 / 500 (1%), though the true error rate is 50%.
+        -  Similarly if the spider crawls 10 archive files, each containing 99 OCDS files.
+        """
+        error_count = self.item_counts["FileError"] + self.logparser["crawler_stats"].get("invalid_json_count", 0)
+        return error_count / (self.item_counts["File"] + error_count)

scrapyloganalyzer-0.0.1/scrapyloganalyzer.egg-info/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: scrapyloganalyzer
+Version: 0.0.1
+Summary: Provides methods to analyze the quality of a Scrapy crawl.
+Author-email: Open Contracting Partnership <data@open-contracting.org>
+License: BSD
+Project-URL: Homepage, https://github.com/open-contracting/scrapy-log-analyzer
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: logparser
+Provides-Extra: test
+Requires-Dist: coverage; extra == "test"
+Requires-Dist: pytest; extra == "test"
+
+|PyPI Version| |Build Status| |Coverage Status| |Python Version|
+
+Provides methods to analyze the quality of a Scrapy crawl.
+
+(If you are viewing this on GitHub, open the `full documentation <https://scrapy-log-analyzer.readthedocs.io/>`__ for additional details.)
+
+.. |PyPI Version| image:: https://img.shields.io/pypi/v/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/
+.. |Build Status| image:: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml/badge.svg
+   :target: https://github.com/open-contracting/scrapy-log-analyzer/actions/workflows/ci.yml
+.. |Coverage Status| image:: https://coveralls.io/repos/github/open-contracting/scrapy-log-analyzer/badge.svg?branch=main
+   :target: https://coveralls.io/github/open-contracting/scrapy-log-analyzer?branch=main
+.. |Python Version| image:: https://img.shields.io/pypi/pyversions/scrapyloganalyzer.svg
+   :target: https://pypi.org/project/scrapyloganalyzer/

scrapyloganalyzer-0.0.1/scrapyloganalyzer.egg-info/SOURCES.txt

@@ -0,0 +1,29 @@
+LICENSE
+MANIFEST.in
+README.rst
+pyproject.toml
+docs/Makefile
+docs/changelog.rst
+docs/conf.py
+docs/index.rst
+docs/requirements.txt
+docs/api/index.rst
+docs/contributing/index.rst
+scrapyloganalyzer/__init__.py
+scrapyloganalyzer.egg-info/PKG-INFO
+scrapyloganalyzer.egg-info/SOURCES.txt
+scrapyloganalyzer.egg-info/dependency_links.txt
+scrapyloganalyzer.egg-info/requires.txt
+scrapyloganalyzer.egg-info/top_level.txt
+tests/__init__.py
+tests/test_scrapy_log_file.py
+tests/fixtures/log1.log
+tests/fixtures/log_crawl_time_crawl_statistic.log
+tests/fixtures/log_crawl_time_log_message.log
+tests/fixtures/log_crawl_time_spider_argument.log
+tests/fixtures/log_error1.log
+tests/fixtures/log_error_invalid_json.log
+tests/fixtures/log_from_date1.log
+tests/fixtures/log_in_progress1.log
+tests/fixtures/log_sample1.log
+tests/fixtures/log_sigint1.log

scrapyloganalyzer-0.0.1/scrapyloganalyzer.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

scrapyloganalyzer-0.0.1/scrapyloganalyzer.egg-info/requires.txt

@@ -0,0 +1,5 @@
+logparser
+
+[test]
+coverage
+pytest

scrapyloganalyzer-0.0.1/scrapyloganalyzer.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+dist
+docs
+scrapyloganalyzer

scrapyloganalyzer-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scrapyloganalyzer-0.0.1/tests/__init__.py

@@ -0,0 +1,5 @@
+import os.path
+
+
+def path(filename):
+    return os.path.join("tests", "fixtures", filename)