data-prep-connector 0.2.2__tar.gz → 0.2.2.dev0__tar.gz

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/Makefile

@@ -2,8 +2,6 @@
 REPOROOT=..
 include $(REPOROOT)/.make.defaults
 
-DPK_CONNECTOR_VERSION=0.2.2
-
 clean::
 	@# Help: Clean up the distribution build and the venv
 	rm -rf dist venv
@@ -15,7 +13,7 @@ clean::
 setup::
 
 set-versions: .check-env
-	$(MAKE) TOML_VERSION=$(DPK_CONNECTOR_VERSION) .defaults.update-toml
+	$(MAKE) TOML_VERSION=$(DPK_LIB_VERSION) .defaults.update-toml
 
 build:: build-dist
 

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_connector
-Version: 0.2.2
+Version: 0.2.2.dev0
 Summary: Scalable and Compliant Web Crawler
 Author-email: Hiroya Matsubara <hmtbr@jp.ibm.com>
 License: Apache-2.0

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/doc/overview.md

@@ -10,20 +10,6 @@ Features:
 - Mime type filters: You can restrict mime types which can be downloaded.
 - Parallel processing: Requests to websites are processed in parallel.
 
-## How to install
-
-### From PyPI
-
-```sh
-pip install data-prep-connector
-```
-
-### From Github
-
-```sh
-pip install git+https://github.com/IBM/data-prep-kit.git@dev#subdirectory=data-connector-lib
-```
-
 ## Example usage
 
 ```python

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_connector"
-version = "0.2.2"
+version = "0.2.2.dev0"
 requires-python = ">=3.10"
 keywords = [
     "data",

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/src/data_prep_connector.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_prep_connector
-Version: 0.2.2
+Version: 0.2.2.dev0
 Summary: Scalable and Compliant Web Crawler
 Author-email: Hiroya Matsubara <hmtbr@jp.ibm.com>
 License: Apache-2.0

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/src/dpk_connector/core/crawler.py

@@ -74,7 +74,6 @@ def async_crawl(
     user_agent: str = "",
     headers: dict[str, str] = {},
     allow_domains: Collection[str] = (),
-    subdomain_focus: bool = False,
     path_focus: bool = False,
     allow_mime_types: Collection[str] = (
         "application/pdf",
@@ -97,7 +96,6 @@ def async_crawl(
         user_agent (str): The user agent string to use for the crawler. Defaults to "Scrapy/VERSION (+https://scrapy.org)".
         headers (dict[str, str]): A dictionary of additional headers to send with each request. Default is an empty dictionary.
         allow_domains (Collection[str]): A collection of domains to restrict the crawler to. Default is the domains of the seed URLs.
-        subdomain_focus (bool): If specified, only links under the subdomains of the input seed URLs will be extracted. Ignored if `allow_domains` is specified.
         path_focus (bool): If specified, only links under the paths of the input seed URLs will be extracted.
         allow_mime_types (Collection[str]): A collection of MIME types to allow during the crawl. Default is a collection containing "application/pdf", "text/html", "text/markdown", and "text/plain".
         disallow_mime_types (Collection[str]): A collection of MIME types to disallow during the crawl. Default is an empty collection.
@@ -142,7 +140,6 @@ def async_crawl(
         seed_urls=seed_urls,
         callback=on_downloaded,
         allow_domains=allow_domains,
-        subdomain_focus=subdomain_focus,
         path_focus=path_focus,
         allow_mime_types=allow_mime_types,
         disallow_mime_types=disallow_mime_types,
@@ -158,7 +155,6 @@ def crawl(
     user_agent: str = "",
     headers: dict[str, str] = {},
     allow_domains: Collection[str] = (),
-    subdomain_focus: bool = False,
     path_focus: bool = False,
     allow_mime_types: Collection[str] = (
         "application/pdf",
@@ -181,7 +177,6 @@ def crawl(
         user_agent (str): The user agent string to use for the crawler. Defaults to "Scrapy/VERSION (+https://scrapy.org)".
         headers (dict[str, str]): A dictionary of additional headers to send with each request. Default is an empty dictionary.
         allow_domains (Collection[str]): A collection of domains to restrict the crawler to. Default is the domains of the seed URLs.
-        subdomain_focus (bool): If specified, only links under the subdomains of the input seed URLs will be extracted. Ignored if `allow_domains` is specified.
         path_focus (bool): If specified, only links under the paths of the input seed URLs will be extracted.
         allow_mime_types (Collection[str]): A collection of MIME types to allow during the crawl. Default is a collection containing "application/pdf", "text/html", "text/markdown", and "text/plain".
         disallow_mime_types (Collection[str]): A collection of MIME types to disallow during the crawl. Default is an empty collection.
@@ -203,7 +198,6 @@ def crawl(
         user_agent,
         headers,
         allow_domains,
-        subdomain_focus,
         path_focus,
         allow_mime_types,
         disallow_mime_types,

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/src/dpk_connector/core/spiders/sitemap.py

@@ -28,7 +28,6 @@ from dpk_connector.core.utils import (
     get_content_type,
     get_etld1,
     get_focus_path,
-    get_fqdn,
     is_allowed_path,
     urlparse_cached,
 )
@@ -43,7 +42,6 @@ class BaseSitemapSpider(SitemapSpider):
         self,
         seed_urls: Collection[str],
         allow_domains: Collection[str] = (),
-        subdomain_focus: bool = False,
         path_focus: bool = False,
         allow_mime_types: Collection[str] = (),
         disallow_mime_types: Collection[str] = (),
@@ -90,15 +88,11 @@ class BaseSitemapSpider(SitemapSpider):
                     self.focus_paths.add(path)
 
         # Domains and mime types filtering
-        if allow_domains:
-            self.allowed_domains = set(allow_domains)
-        elif subdomain_focus:
-            self.allowed_domains = set()
-            for url in seed_urls:
-                if fqdn := get_fqdn(url):
-                    self.allowed_domains.add(fqdn)
-        else:
-            self.allowed_domains = set(get_etld1(url) for url in seed_urls)
+        self.allowed_domains = set(
+            allow_domains
+            if len(allow_domains) > 0
+            else [get_etld1(url) for url in seed_urls]
+        )
         self.allow_mime_types = set(
             [m.lower() for m in allow_mime_types] if len(allow_mime_types) > 0 else ()
         )
@@ -161,9 +155,7 @@ class BaseSitemapSpider(SitemapSpider):
             )
 
     def _parse_sitemap(self, response: Response):
-        yield ConnectorItem(
-            dropped=False, downloaded=False, system_request=True, sitemap=True
-        )
+        yield ConnectorItem(dropped=False, downloaded=False, system_request=True, sitemap=True)
 
         seed_url = response.meta["seed_url"]
 

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/src/dpk_connector/core/utils.py

@@ -57,11 +57,6 @@ def get_etld1(url: str) -> str:
     return f"{ext.domain}.{ext.suffix}"
 
 
-def get_fqdn(url: str) -> str:
-    ext = tldextract.extract(url)
-    return ext.fqdn
-
-
 def get_focus_path(url: str) -> str | None:
     parts = urlparse_cached(url)
     if len(parts.path.split("/")) > 2:

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/test/dpk_connector/core/test_crawler.py

@@ -1,16 +1,5 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
 import pytest
+
 from dpk_connector.core.crawler import crawl
 
 

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/test/dpk_connector/core/test_middlewares.py

@@ -1,15 +1,3 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
 import pytest
 from dpk_connector.core.middlewares import DelayingProtegoRobotParser
 from pytest_mock import MockerFixture

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/test/dpk_connector/core/test_sitemap_spider.py

@@ -1,24 +1,13 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
 from pathlib import Path
 
 import pytest
-from dpk_connector.core.item import ConnectorItem
-from dpk_connector.core.spiders.sitemap import BaseSitemapSpider, ConnectorSitemapSpider
 from scrapy import Request
 from scrapy.crawler import Crawler
 from scrapy.http import HtmlResponse
 
+from dpk_connector.core.item import ConnectorItem
+from dpk_connector.core.spiders.sitemap import BaseSitemapSpider, ConnectorSitemapSpider
+
 
 @pytest.fixture
 def crawler() -> Crawler:
@@ -33,21 +22,6 @@ def crawler() -> Crawler:
     return crawler
 
 
-def test_init_subdomain_focus():
-    spider = BaseSitemapSpider(
-        seed_urls=(
-            "http://blog.example.com/",
-            "http://contents.example.com/",
-        ),
-        subdomain_focus=True,
-    )
-    assert spider.seed_urls == {
-        "http://blog.example.com/",
-        "http://contents.example.com/",
-    }
-    assert spider.allowed_domains == {"blog.example.com", "contents.example.com"}
-
-
 def test_init_path_focus():
     spider = BaseSitemapSpider(
         seed_urls=(
@@ -85,7 +59,9 @@ def test_parse(datadir: Path, crawler: Crawler):
         assert body.decode("utf-8") == response_body
         assert headers == {"Content-Type": "text/html"}
 
-    spider = ConnectorSitemapSpider.from_crawler(crawler, seed_urls=("http://example.com",), callback=callback)
+    spider = ConnectorSitemapSpider.from_crawler(
+        crawler, seed_urls=("http://example.com",), callback=callback
+    )
     request = Request(
         "http://example.com/index.html",
         meta={
@@ -103,7 +79,9 @@ def test_parse(datadir: Path, crawler: Crawler):
     parsed = spider.parse(response)
 
     item = next(parsed)
-    assert item == ConnectorItem(dropped=False, downloaded=True, system_request=False, sitemap=False)
+    assert item == ConnectorItem(
+        dropped=False, downloaded=True, system_request=False, sitemap=False
+    )
 
     for next_request in parsed:
         assert isinstance(next_request, Request) is True

{data_prep_connector-0.2.2 → data_prep_connector-0.2.2.dev0}/test/dpk_connector/core/test_utils.py

@@ -1,15 +1,3 @@
-# (C) Copyright IBM Corp. 2024.
-# Licensed under the Apache License, Version 2.0 (the “License”);
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an “AS IS” BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
 # Assisted by WCA@IBM
 # Latest GenAI contribution: ibm/granite-20b-code-instruct-v2
 
@@ -19,7 +7,6 @@ from dpk_connector.core.utils import (
     get_content_type,
     get_etld1,
     get_focus_path,
-    get_fqdn,
     get_header_value,
     get_mime_type,
     is_allowed_path,
@@ -32,7 +19,9 @@ from scrapy.http import Request, Response
 
 
 def test_get_header_value():
-    response = Response("http://example.com", headers={"Content-Type": "application/json"})
+    response = Response(
+        "http://example.com", headers={"Content-Type": "application/json"}
+    )
     assert get_header_value(response, "Content-Type") == "application/json"
 
 
@@ -94,21 +83,6 @@ def test_get_etld1(url: str, expected: str):
     assert get_etld1(url) == expected
 
 
-@pytest.mark.parametrize(
-    "url,expected",
-    [
-        ("http://www.example.com", "www.example.com"),
-        ("https://www.example.co.uk", "www.example.co.uk"),
-        ("http://www.example.com/path?query=string#fragment", "www.example.com"),
-        ("http://localhost:8080/", ""),
-        ("http://www.example.com:8080/", "www.example.com"),
-        ("http://www.sub.example.com:8080/", "www.sub.example.com"),
-    ],
-)
-def test_get_fqdn(url: str, expected: str):
-    assert get_fqdn(url) == expected
-
-
 @pytest.mark.parametrize(
     "url,expected",
     [