data-prep-connector 0.2.2__tar.gz

data_prep_connector-0.2.2/Makefile

@@ -0,0 +1,51 @@
+# Use make help, to see the available rules
+REPOROOT=..
+include $(REPOROOT)/.make.defaults
+
+DPK_CONNECTOR_VERSION=0.2.2
+
+clean::
+	@# Help: Clean up the distribution build and the venv
+	rm -rf dist venv
+	rm -rf src/*egg-info
+
+.check-env::
+	@echo "Checks passed"
+
+setup::
+
+set-versions: .check-env
+	$(MAKE) TOML_VERSION=$(DPK_CONNECTOR_VERSION) .defaults.update-toml
+
+build:: build-dist
+
+#build:: update-toml .defaults.build-dist
+build-dist :: .defaults.build-dist
+
+publish:: publish-dist
+
+publish-dist :: .check-env .defaults.publish-dist
+
+venv::	pyproject.toml
+	@# Help: Create the virtual environment using pyproject.toml
+	rm -r dist venv || true
+	rm -rf src/*egg-info || true
+	rm makeenv || true
+	$(PYTHON) -m venv venv
+	source venv/bin/activate; 	\
+	pip install --upgrade pip;	\
+	pip install -e .;		\
+	pip install pytest pytest-mock pytest-datadir pytest-cov moto==5.0.5 markupsafe==2.0.1
+
+image::
+	@# Help: Placeholder does nothing for now.
+	@echo "Image building for ray is in the works (comming soon)."
+
+# Here we run each test directory of tests and each ray launched test separately, because
+# it seems when running multiple ray launch tests in a single pytest run there is some sort of ray.init() duplication.
+# pytest-forked was tried, but then we get SIGABRT in pytest when running the s3 tests, some of which are skipped..
+# TODO: the following fails.  Why?  source venv/bin/activate; export PYTHONPATH=../src; cd test; $(PYTEST)  .
+.PHONY: test
+test::   venv
+	@# Help: Use the already-built virtual environment to run pytest on the test directory.
+	source venv/bin/activate; $(PYTEST);

data_prep_connector-0.2.2/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.1
+Name: data_prep_connector
+Version: 0.2.2
+Summary: Scalable and Compliant Web Crawler
+Author-email: Hiroya Matsubara <hmtbr@jp.ibm.com>
+License: Apache-2.0
+Keywords: data,data acquisition,crawler,web crawler,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: scrapy>=2.11.2
+Requires-Dist: pydantic>=2.8.1
+Requires-Dist: tldextract>=5.1.2
+Provides-Extra: dev
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: pytest>=7.3.2; extra == "dev"
+Requires-Dist: pytest-dotenv>=0.5.2; extra == "dev"
+Requires-Dist: pytest-env>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.3.2; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
+Requires-Dist: pytest-datadir>=1.5.0; extra == "dev"
+Requires-Dist: moto==5.0.5; extra == "dev"
+Requires-Dist: markupsafe==2.0.1; extra == "dev"
+
+# DPK Connector
+
+DPK Connector is a scalable and compliant web crawler developed for data acquisition towards LLM development. It is built on [Scrapy](https://scrapy.org/).
+For more details read [the documentation](doc/overview.md).
+
+## Virtual Environment
+
+The project uses `pyproject.toml` and a Makefile for operations.
+To do development you should establish the virtual environment
+```shell
+make venv
+```
+and then either activate
+```shell
+source venv/bin/activate
+```
+or set up your IDE to use the venv directory when developing in this project
+
+## Library Artifact Build and Publish
+
+To test, build and publish the library
+```shell
+make test build publish
+```
+
+To up the version number, edit the Makefile to change VERSION and rerun the above. This will require committing both the `Makefile` and the autotmatically updated `pyproject.toml` file.
+
+## How to use
+
+See [the overview](doc/overview.md).

data_prep_connector-0.2.2/README.md

@@ -0,0 +1,30 @@
+# DPK Connector
+
+DPK Connector is a scalable and compliant web crawler developed for data acquisition towards LLM development. It is built on [Scrapy](https://scrapy.org/).
+For more details read [the documentation](doc/overview.md).
+
+## Virtual Environment
+
+The project uses `pyproject.toml` and a Makefile for operations.
+To do development you should establish the virtual environment
+```shell
+make venv
+```
+and then either activate
+```shell
+source venv/bin/activate
+```
+or set up your IDE to use the venv directory when developing in this project
+
+## Library Artifact Build and Publish
+
+To test, build and publish the library
+```shell
+make test build publish
+```
+
+To up the version number, edit the Makefile to change VERSION and rerun the above. This will require committing both the `Makefile` and the autotmatically updated `pyproject.toml` file.
+
+## How to use
+
+See [the overview](doc/overview.md).

data_prep_connector-0.2.2/doc/overview.md

@@ -0,0 +1,61 @@
+# DPK Connector Overview
+
+The Data Prep Kit Connector (DPK Connector) is a Python library for scalable and compliant web crawling.
+
+Features:
+- Robots.txt compliant: The Connector follows allow/disallow lists and some extended directives such as `Crawl-delay` in robots.txt of websites.
+- Sitemap support: The Connector automatically parses sitemap urls from input and tries to find them from robots.txt.
+- User agent and headers customization: You can use your own user agent string and request headers.
+- Domain and path focus: You can limit domains and paths accessed by the library.
+- Mime type filters: You can restrict mime types which can be downloaded.
+- Parallel processing: Requests to websites are processed in parallel.
+
+## How to install
+
+### From PyPI
+
+```sh
+pip install data-prep-connector
+```
+
+### From Github
+
+```sh
+pip install git+https://github.com/IBM/data-prep-kit.git@dev#subdirectory=data-connector-lib
+```
+
+## Example usage
+
+```python
+from dpk_connector import crawl, shutdown
+
+
+def main():
+    """
+    An example of running a crawl.
+    """
+
+    def on_downloaded(url: str, body: bytes, headers: dict) -> None:
+        """
+        Callback function called when a page has been downloaded.
+        You have access to the request URL, response body and headers.
+        """
+        print(f"url: {url}, headers: {headers}, body: {body[:64]}")
+
+    user_agent = "Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0"
+
+    # Start crawling
+    crawl(
+        ["https://crawler-test.com/"],
+        on_downloaded,
+        user_agent=user_agent,
+        depth_limit=0,
+    )  # blocking call
+
+    # Shutdown all crawls
+    shutdown()
+
+
+if __name__ == "__main__":
+    main()
+```

data_prep_connector-0.2.2/pyproject.toml

@@ -0,0 +1,61 @@
+[project]
+name = "data_prep_connector"
+version = "0.2.2"
+requires-python = ">=3.10"
+keywords = [
+    "data",
+    "data acquisition",
+    "crawler",
+    "web crawler",
+    "llm",
+    "generative",
+    "ai",
+    "fine-tuning",
+    "llmapps",
+]
+description = "Scalable and Compliant Web Crawler"
+license = { text = "Apache-2.0" }
+readme = { file = "README.md", content-type = "text/markdown" }
+authors = [{ name = "Hiroya Matsubara", email = "hmtbr@jp.ibm.com" }]
+dependencies = [
+    "scrapy>=2.11.2",
+    "pydantic>=2.8.1",
+    "tldextract>=5.1.2",
+]
+
+[project_urls]
+Repository = "https://github.com/IBM/data-prep-kit"
+Issues = "https://github.com/IBM/data-prep-kit/issues"
+Documentation = "https://ibm.github.io/data-prep-kit/"
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+[project.optional-dependencies]
+dev = [
+    "twine",
+    "pytest>=7.3.2",
+    "pytest-dotenv>=0.5.2",
+    "pytest-env>=1.0.0",
+    "pre-commit>=3.3.2",
+    "pytest-cov>=4.1.0",
+    "pytest-mock>=3.10.0",
+    "pytest-datadir>=1.5.0",
+    "moto==5.0.5",
+    "markupsafe==2.0.1",
+]
+
+[options]
+package_dir = ["src", "test"]
+
+[options.packages.find]
+where = ["src/dpk_connector"]
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]

data_prep_connector-0.2.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

data_prep_connector-0.2.2/src/data_prep_connector.egg-info/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.1
+Name: data_prep_connector
+Version: 0.2.2
+Summary: Scalable and Compliant Web Crawler
+Author-email: Hiroya Matsubara <hmtbr@jp.ibm.com>
+License: Apache-2.0
+Keywords: data,data acquisition,crawler,web crawler,llm,generative,ai,fine-tuning,llmapps
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: scrapy>=2.11.2
+Requires-Dist: pydantic>=2.8.1
+Requires-Dist: tldextract>=5.1.2
+Provides-Extra: dev
+Requires-Dist: twine; extra == "dev"
+Requires-Dist: pytest>=7.3.2; extra == "dev"
+Requires-Dist: pytest-dotenv>=0.5.2; extra == "dev"
+Requires-Dist: pytest-env>=1.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.3.2; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.10.0; extra == "dev"
+Requires-Dist: pytest-datadir>=1.5.0; extra == "dev"
+Requires-Dist: moto==5.0.5; extra == "dev"
+Requires-Dist: markupsafe==2.0.1; extra == "dev"
+
+# DPK Connector
+
+DPK Connector is a scalable and compliant web crawler developed for data acquisition towards LLM development. It is built on [Scrapy](https://scrapy.org/).
+For more details read [the documentation](doc/overview.md).
+
+## Virtual Environment
+
+The project uses `pyproject.toml` and a Makefile for operations.
+To do development you should establish the virtual environment
+```shell
+make venv
+```
+and then either activate
+```shell
+source venv/bin/activate
+```
+or set up your IDE to use the venv directory when developing in this project
+
+## Library Artifact Build and Publish
+
+To test, build and publish the library
+```shell
+make test build publish
+```
+
+To up the version number, edit the Makefile to change VERSION and rerun the above. This will require committing both the `Makefile` and the autotmatically updated `pyproject.toml` file.
+
+## How to use
+
+See [the overview](doc/overview.md).

data_prep_connector-0.2.2/src/data_prep_connector.egg-info/SOURCES.txt

@@ -0,0 +1,26 @@
+Makefile
+README.md
+pyproject.toml
+doc/overview.md
+src/data_prep_connector.egg-info/PKG-INFO
+src/data_prep_connector.egg-info/SOURCES.txt
+src/data_prep_connector.egg-info/dependency_links.txt
+src/data_prep_connector.egg-info/requires.txt
+src/data_prep_connector.egg-info/top_level.txt
+src/dpk_connector/__init__.py
+src/dpk_connector/core/__init__.py
+src/dpk_connector/core/crawler.py
+src/dpk_connector/core/item.py
+src/dpk_connector/core/logging.py
+src/dpk_connector/core/middlewares.py
+src/dpk_connector/core/pipelines.py
+src/dpk_connector/core/settings.py
+src/dpk_connector/core/utils.py
+src/dpk_connector/core/spiders/__init__.py
+src/dpk_connector/core/spiders/sitemap.py
+test/dpk_connector/core/__init__.py
+test/dpk_connector/core/test_crawler.py
+test/dpk_connector/core/test_middlewares.py
+test/dpk_connector/core/test_sitemap_spider.py
+test/dpk_connector/core/test_utils.py
+test/dpk_connector/core/test_sitemap_spider/index.html

data_prep_connector-0.2.2/src/data_prep_connector.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data_prep_connector-0.2.2/src/data_prep_connector.egg-info/requires.txt

@@ -0,0 +1,15 @@
+scrapy>=2.11.2
+pydantic>=2.8.1
+tldextract>=5.1.2
+
+[dev]
+twine
+pytest>=7.3.2
+pytest-dotenv>=0.5.2
+pytest-env>=1.0.0
+pre-commit>=3.3.2
+pytest-cov>=4.1.0
+pytest-mock>=3.10.0
+pytest-datadir>=1.5.0
+moto==5.0.5
+markupsafe==2.0.1

data_prep_connector-0.2.2/src/data_prep_connector.egg-info/top_level.txt

@@ -0,0 +1 @@
+dpk_connector

data_prep_connector-0.2.2/src/dpk_connector/__init__.py

@@ -0,0 +1,13 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from dpk_connector.core.crawler import async_crawl, crawl, shutdown  # noqa

data_prep_connector-0.2.2/src/dpk_connector/core/__init__.py

@@ -0,0 +1,11 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################

data_prep_connector-0.2.2/src/dpk_connector/core/crawler.py

@@ -0,0 +1,222 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import threading
+from typing import Any, Callable, Collection, Type, cast
+
+from scrapy import Spider
+from scrapy.crawler import Crawler, CrawlerRunner
+from scrapy.settings import Settings
+from twisted.internet.defer import Deferred
+
+from dpk_connector.core.utils import validate_domain, validate_url
+
+_lock = threading.Lock()
+_reactor_initialized = False
+_reactor_started = False
+
+
+def _run_reactor():
+    from twisted.internet import reactor
+
+    reactor.run(installSignalHandlers=False)
+
+
+_reactor_thread: threading.Thread = threading.Thread(
+    target=_run_reactor,
+    daemon=True,
+)
+
+
+def _start_reactor():
+    with _lock:
+        global _reactor_started
+        if not _reactor_started:
+            _reactor_thread.start()
+            _reactor_started = True
+
+
+def _stop_reactor():
+    from twisted.internet import reactor
+
+    try:
+        reactor.stop()
+    except RuntimeError:
+        pass
+
+
+class MultiThreadedCrawlerRunner(CrawlerRunner):
+    def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler:
+        if isinstance(spidercls, str):
+            spidercls = self.spider_loader.load(spidercls)
+        with _lock:
+            global _reactor_initialized
+            init_reactor = not _reactor_initialized
+            crawler = Crawler(
+                cast(Type[Spider], spidercls), self.settings, init_reactor
+            )
+            _reactor_initialized = True
+        return crawler
+
+
+def async_crawl(
+    seed_urls: Collection[str],
+    on_downloaded: Callable[[str, bytes, dict[str, str]], None],
+    user_agent: str = "",
+    headers: dict[str, str] = {},
+    allow_domains: Collection[str] = (),
+    subdomain_focus: bool = False,
+    path_focus: bool = False,
+    allow_mime_types: Collection[str] = (
+        "application/pdf",
+        "text/html",
+        "text/markdown",
+        "text/plain",
+    ),
+    disallow_mime_types: Collection[str] = (),
+    depth_limit: int = -1,
+    download_limit: int = -1,
+) -> Deferred[None]:
+    # Assisted by WCA@IBM
+    # Latest GenAI contribution: ibm/granite-20b-code-instruct-v2
+    """
+    Do crawl asynchronously.
+
+    Parameters:
+        seed_urls (Collection[str]): A collection of seed URLs to start the crawl from.
+        on_downloaded (Callable[[str, bytes, dict[str, str]], None]): The callback function to be called for each downloaded page.
+        user_agent (str): The user agent string to use for the crawler. Defaults to "Scrapy/VERSION (+https://scrapy.org)".
+        headers (dict[str, str]): A dictionary of additional headers to send with each request. Default is an empty dictionary.
+        allow_domains (Collection[str]): A collection of domains to restrict the crawler to. Default is the domains of the seed URLs.
+        subdomain_focus (bool): If specified, only links under the subdomains of the input seed URLs will be extracted. Ignored if `allow_domains` is specified.
+        path_focus (bool): If specified, only links under the paths of the input seed URLs will be extracted.
+        allow_mime_types (Collection[str]): A collection of MIME types to allow during the crawl. Default is a collection containing "application/pdf", "text/html", "text/markdown", and "text/plain".
+        disallow_mime_types (Collection[str]): A collection of MIME types to disallow during the crawl. Default is an empty collection.
+        depth_limit (int): The maximum depth of the crawl. Default is -1, which means no limit.
+        download_limit (int): The maximum number of pages to download. Default is -1, which means no limit. This is a soft limit, meaning that a crawler may download more pages than the specified limit.
+
+    Returns:
+        Deferred[None]: A Twisted deferred object that can be used to wait for the crawler to finish.
+    """
+    if not seed_urls:
+        raise ValueError(f"Empty seed URLs.")
+    for url in seed_urls:
+        if not validate_url(url):
+            raise ValueError(f"Seed URL {url} is not valid.")
+    for domain in allow_domains:
+        if not validate_domain(domain):
+            raise ValueError(f"Allow domain {domain} is not valid.")
+    if depth_limit < -1:
+        raise ValueError(f"Invalid depth limit {depth_limit}")
+    if download_limit < -1:
+        raise ValueError(f"Invalid download limit {download_limit}")
+
+    settings = Settings()
+    settings.setmodule("dpk_connector.core.settings", priority="project")
+
+    if user_agent:
+        settings.set("USER_AGENT", user_agent, priority="spider")
+    if headers:
+        settings.set("DEFAULT_REQUEST_HEADERS", headers)
+    if depth_limit == 0:
+        depth_limit = -1
+    elif depth_limit == -1:
+        depth_limit = 0
+    settings.set("DEPTH_LIMIT", depth_limit, priority="spider")
+    if download_limit == -1:
+        download_limit = 0
+    settings.set("CLOSESPIDER_ITEMCOUNT", download_limit, priority="spider")
+
+    runner = MultiThreadedCrawlerRunner(settings)
+    runner.crawl(
+        "dpk-connector-sitemap",
+        seed_urls=seed_urls,
+        callback=on_downloaded,
+        allow_domains=allow_domains,
+        subdomain_focus=subdomain_focus,
+        path_focus=path_focus,
+        allow_mime_types=allow_mime_types,
+        disallow_mime_types=disallow_mime_types,
+        disable_sitemap_search=True,
+    )
+    _start_reactor()
+    return runner.join()
+
+
+def crawl(
+    seed_urls: Collection[str],
+    on_downloaded: Callable[[str, bytes, dict[str, str]], None],
+    user_agent: str = "",
+    headers: dict[str, str] = {},
+    allow_domains: Collection[str] = (),
+    subdomain_focus: bool = False,
+    path_focus: bool = False,
+    allow_mime_types: Collection[str] = (
+        "application/pdf",
+        "text/html",
+        "text/markdown",
+        "text/plain",
+    ),
+    disallow_mime_types: Collection[str] = (),
+    depth_limit: int = -1,
+    download_limit: int = -1,
+) -> None:
+    # Assisted by WCA@IBM
+    # Latest GenAI contribution: ibm/granite-20b-code-instruct-v2
+    """
+    Do crawl synchronously.
+
+    Parameters:
+        seed_urls (Collection[str]): A collection of seed URLs to start the crawl from.
+        on_downloaded (Callable[[str, bytes, dict[str, str]], None]): The callback function to be called for each downloaded page.
+        user_agent (str): The user agent string to use for the crawler. Defaults to "Scrapy/VERSION (+https://scrapy.org)".
+        headers (dict[str, str]): A dictionary of additional headers to send with each request. Default is an empty dictionary.
+        allow_domains (Collection[str]): A collection of domains to restrict the crawler to. Default is the domains of the seed URLs.
+        subdomain_focus (bool): If specified, only links under the subdomains of the input seed URLs will be extracted. Ignored if `allow_domains` is specified.
+        path_focus (bool): If specified, only links under the paths of the input seed URLs will be extracted.
+        allow_mime_types (Collection[str]): A collection of MIME types to allow during the crawl. Default is a collection containing "application/pdf", "text/html", "text/markdown", and "text/plain".
+        disallow_mime_types (Collection[str]): A collection of MIME types to disallow during the crawl. Default is an empty collection.
+        depth_limit (int): The maximum depth of the crawl. Default is -1, which means no limit.
+        download_limit (int): The maximum number of pages to download. Default is -1, which means no limit. This is a soft limit, meaning that a crawler may download more pages than the specified limit.
+
+    Returns:
+        None
+    """
+    condition = threading.Condition()
+
+    def on_completed(result: Any):
+        with condition:
+            condition.notify()
+
+    d = async_crawl(
+        seed_urls,
+        on_downloaded,
+        user_agent,
+        headers,
+        allow_domains,
+        subdomain_focus,
+        path_focus,
+        allow_mime_types,
+        disallow_mime_types,
+        depth_limit,
+        download_limit,
+    )
+    d.addBoth(on_completed)
+    with condition:
+        condition.wait()
+
+
+def shutdown():
+    """
+    Shutdown all crawls.
+    """
+    _stop_reactor()

data_prep_connector-0.2.2/src/dpk_connector/core/item.py

@@ -0,0 +1,21 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from dataclasses import dataclass
+
+
+@dataclass
+class ConnectorItem:
+    dropped: bool = False
+    downloaded: bool = False
+    system_request: bool = False
+    sitemap: bool = False

data_prep_connector-0.2.2/src/dpk_connector/core/logging.py

@@ -0,0 +1,22 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from scrapy.logformatter import LogFormatter as ScrapyLogFormatter
+
+
+class QuietLogFormatter(ScrapyLogFormatter):
+    def scraped(self, item, response, spider):
+        return (
+            super().scraped(item, response, spider)
+            if spider.settings.getbool("LOG_SCRAPED_ITEMS")
+            else None
+        )