crawlee 1.0.5b3__tar.gz → 1.0.5b7__tar.gz

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 ### 🐛 Bug Fixes
 
 - Improve indexing of the `request_queue_records` table for `SqlRequestQueueClient` ([#1527](https://github.com/apify/crawlee-python/pull/1527)) ([6509534](https://github.com/apify/crawlee-python/commit/65095346a9d8b703b10c91e0510154c3c48a4176)) by [@Mantisus](https://github.com/Mantisus), closes [#1526](https://github.com/apify/crawlee-python/issues/1526)
+- Improve error handling for `RobotsTxtFile.load` ([#1524](https://github.com/apify/crawlee-python/pull/1524)) ([596a311](https://github.com/apify/crawlee-python/commit/596a31184914a254b3e7a81fd2f48ea8eda7db49)) by [@Mantisus](https://github.com/Mantisus)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b3
+Version: 1.0.5b7
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/docs/pyproject.toml

@@ -1,4 +1,4 @@
-# Line lenght different from the rest of the code to make sure that the example codes visualised on the generated
+# Line length different from the rest of the code to make sure that the example codes visualised on the generated
 # documentation webpages are shown without vertical slider to make them more readable.
 
 [tool.ruff]

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.5b3"
+version = "1.0.5b7"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/src/crawlee/_utils/robots.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from protego import Protego
@@ -15,6 +16,9 @@ if TYPE_CHECKING:
     from crawlee.proxy_configuration import ProxyInfo
 
 
+logger = getLogger(__name__)
+
+
 class RobotsTxtFile:
     def __init__(
         self, url: str, robots: Protego, http_client: HttpClient | None = None, proxy_info: ProxyInfo | None = None
@@ -56,12 +60,20 @@ class RobotsTxtFile:
             http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file.
             proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
         """
-        response = await http_client.send_request(url, proxy_info=proxy_info)
-        body = (
-            b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else await response.read()
-        )
+        try:
+            response = await http_client.send_request(url, proxy_info=proxy_info)
+
+            body = (
+                b'User-agent: *\nAllow: /'
+                if is_status_code_client_error(response.status_code)
+                else await response.read()
+            )
+            robots = Protego.parse(body.decode('utf-8'))
+
+        except Exception as e:
+            logger.warning(f'Failed to fetch from robots.txt from "{url}" with error: "{e}"')
 
-        robots = Protego.parse(body.decode('utf-8'))
+            robots = Protego.parse('User-agent: *\nAllow: /')
 
         return cls(url, robots, http_client=http_client, proxy_info=proxy_info)
 

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/src/crawlee/statistics/_error_snapshotter.py

@@ -32,7 +32,7 @@ class ErrorSnapshotter:
         """Capture error snapshot and save it to key value store.
 
         It saves the error snapshot directly to a key value store. It can't use `context.get_key_value_store` because
-        it returns `KeyValueStoreChangeRecords` which is commited to the key value store only if the `RequestHandler`
+        it returns `KeyValueStoreChangeRecords` which is committed to the key value store only if the `RequestHandler`
         returned without an exception. ErrorSnapshotter is on the contrary active only when `RequestHandler` fails with
         an exception.
 

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -292,7 +292,7 @@ async def test_playwright_only_hook(test_urls: list[str]) -> None:
 
     await crawler.run(test_urls[:1])
 
-    # Default behavior. Hook is called everytime, both static sub crawler and playwright sub crawler.
+    # Default behavior. Hook is called every time, both static sub crawler and playwright sub crawler.
     pre_nav_hook_common.assert_has_calls([call(test_urls[0]), call(test_urls[0])])
     # Hook is called only by playwright sub crawler.
     pre_nav_hook_playwright.assert_called_once_with('about:blank')
@@ -433,13 +433,13 @@ async def test_adaptive_crawling_statistics(test_urls: list[str]) -> None:
     ],
 )
 async def test_adaptive_crawler_exceptions_in_sub_crawlers(*, error_in_pw_crawler: bool, test_urls: list[str]) -> None:
-    """Test that correct results are commited when exceptions are raised in sub crawlers.
+    """Test that correct results are committed when exceptions are raised in sub crawlers.
 
     Exception in bs sub crawler will be logged and pw sub crawler used instead.
     Any result from bs sub crawler will be discarded, result form pw crawler will be saved instead.
     (But global state modifications through `use_state` will not be reverted!!!)
 
-    Exception in pw sub crawler will prevent any result from being commited. Even if `push_data` was called before
+    Exception in pw sub crawler will prevent any result from being committed. Even if `push_data` was called before
     the exception
     """
     static_only_no_detection_predictor = _SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([0]))

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -6,7 +6,7 @@ from unittest import mock
 import pytest
 
 from crawlee import ConcurrencySettings, Glob, HttpHeaders, RequestTransformAction, SkippedReason
-from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
+from crawlee.crawlers import BasicCrawlingContext, BeautifulSoupCrawler, BeautifulSoupCrawlingContext
 from crawlee.storages import RequestQueue
 
 if TYPE_CHECKING:
@@ -167,6 +167,40 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     }
 
 
+async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None:
+    """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    visit = mock.Mock()
+    fail = mock.Mock()
+    crawler = BeautifulSoupCrawler(
+        http_client=http_client,
+        respect_robots_txt_file=True,
+        max_request_retries=0,
+    )
+
+    @crawler.router.default_handler
+    async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
+        visit(context.request.url)
+        await context.enqueue_links(strategy='all')
+
+    @crawler.failed_request_handler
+    async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
+        fail(context.request.url)
+
+    await crawler.run([str(server_url / 'problematic_links')])
+
+    visited = {call[0][0] for call in visit.call_args_list}
+    failed = {call[0][0] for call in fail.call_args_list}
+
+    # Email must be skipped
+    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    assert visited == {str(server_url / 'problematic_links'), 'https://avatars.githubusercontent.com/apify'}
+
+    # The budplaceholder.com does not exist.
+    assert failed == {
+        'https://budplaceholder.com/',
+    }
+
+
 async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> None:
     crawler = BeautifulSoupCrawler(http_client=http_client, respect_robots_txt_file=True)
     skip = mock.Mock()

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/crawlers/_http/test_http_crawler.py

@@ -126,9 +126,9 @@ async def test_handles_client_errors(
         pytest.param([], True, 4, 1, id='default_behavior'),
         # error without retry for all 4xx statuses
         pytest.param([], False, 0, 1, id='default_behavior_without_session_pool'),
-        # take as successful status codes from the `ignore_http_error_status_codes` list with Sessoion Pool
+        # take as successful status codes from the `ignore_http_error_status_codes` list with Session Pool
         pytest.param([403], True, 0, 0, id='ignore_error_status_codes'),
-        # take as successful status codes from the `ignore_http_error_status_codes` list without Sessoion Pool
+        # take as successful status codes from the `ignore_http_error_status_codes` list without Session Pool
         pytest.param([403], False, 0, 0, id='ignore_error_status_codes_without_session_pool'),
     ],
 )

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -14,7 +14,7 @@ if TYPE_CHECKING:
     from yarl import URL
 
     from crawlee._request import RequestOptions
-    from crawlee.crawlers import ParselCrawlingContext
+    from crawlee.crawlers import BasicCrawlingContext, ParselCrawlingContext
     from crawlee.http_clients._base import HttpClient
 
 
@@ -261,6 +261,40 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     }
 
 
+async def test_respect_robots_txt_with_problematic_links(server_url: URL, http_client: HttpClient) -> None:
+    """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    visit = mock.Mock()
+    fail = mock.Mock()
+    crawler = ParselCrawler(
+        http_client=http_client,
+        respect_robots_txt_file=True,
+        max_request_retries=0,
+    )
+
+    @crawler.router.default_handler
+    async def request_handler(context: ParselCrawlingContext) -> None:
+        visit(context.request.url)
+        await context.enqueue_links(strategy='all')
+
+    @crawler.failed_request_handler
+    async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
+        fail(context.request.url)
+
+    await crawler.run([str(server_url / 'problematic_links')])
+
+    visited = {call[0][0] for call in visit.call_args_list}
+    failed = {call[0][0] for call in fail.call_args_list}
+
+    # Email must be skipped
+    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    assert visited == {str(server_url / 'problematic_links'), 'https://avatars.githubusercontent.com/apify'}
+
+    # The budplaceholder.com does not exist.
+    assert failed == {
+        'https://budplaceholder.com/',
+    }
+
+
 async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> None:
     crawler = ParselCrawler(http_client=http_client, respect_robots_txt_file=True)
     skip = mock.Mock()

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -48,7 +48,7 @@ if TYPE_CHECKING:
     from crawlee._request import RequestOptions
     from crawlee._types import HttpMethod, HttpPayload
     from crawlee.browsers._types import BrowserType
-    from crawlee.crawlers import PlaywrightCrawlingContext, PlaywrightPreNavCrawlingContext
+    from crawlee.crawlers import BasicCrawlingContext, PlaywrightCrawlingContext, PlaywrightPreNavCrawlingContext
 
 
 @pytest.mark.parametrize(
@@ -671,6 +671,39 @@ async def test_respect_robots_txt(server_url: URL) -> None:
     }
 
 
+async def test_respect_robots_txt_with_problematic_links(server_url: URL) -> None:
+    """Test checks the crawler behavior with links that may cause problems when attempting to retrieve robots.txt."""
+    visit = mock.Mock()
+    fail = mock.Mock()
+    crawler = PlaywrightCrawler(
+        respect_robots_txt_file=True,
+        max_request_retries=0,
+    )
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        visit(context.request.url)
+        await context.enqueue_links(strategy='all')
+
+    @crawler.failed_request_handler
+    async def error_handler(context: BasicCrawlingContext, _error: Exception) -> None:
+        fail(context.request.url)
+
+    await crawler.run([str(server_url / 'problematic_links')])
+
+    visited = {call[0][0] for call in visit.call_args_list}
+    failed = {call[0][0] for call in fail.call_args_list}
+
+    # Email must be skipped
+    # https://avatars.githubusercontent.com/apify does not get robots.txt, but is correct for the crawler.
+    assert visited == {str(server_url / 'problematic_links'), 'https://avatars.githubusercontent.com/apify'}
+
+    # The budplaceholder.com does not exist.
+    assert failed == {
+        'https://budplaceholder.com/',
+    }
+
+
 async def test_on_skipped_request(server_url: URL) -> None:
     crawler = PlaywrightCrawler(respect_robots_txt_file=True)
     skip = mock.Mock()

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/server.py

@@ -18,6 +18,7 @@ from tests.unit.server_endpoints import (
     GENERIC_RESPONSE,
     HELLO_WORLD,
     INCAPSULA,
+    PROBLEMATIC_LINKS,
     ROBOTS_TXT,
     SECONDARY_INDEX,
     START_ENQUEUE,
@@ -102,6 +103,7 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
         'page_1': generic_response_endpoint,
         'page_2': generic_response_endpoint,
         'page_3': generic_response_endpoint,
+        'problematic_links': problematic_links_endpoint,
         'set_cookies': set_cookies,
         'set_complex_cookies': set_complex_cookies,
         'cookies': get_cookies,
@@ -287,6 +289,14 @@ async def generic_response_endpoint(_scope: dict[str, Any], _receive: Receive, s
     )
 
 
+async def problematic_links_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    """Handle requests with a page containing problematic links."""
+    await send_html_response(
+        send,
+        PROBLEMATIC_LINKS,
+    )
+
+
 async def redirect_to_url(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
     """Handle requests that should redirect to a specified full URL."""
     query_params = get_query_params(scope.get('query_string', b''))

{crawlee-1.0.5b3 → crawlee-1.0.5b7}/tests/unit/server_endpoints.py

@@ -35,6 +35,16 @@ INCAPSULA = b"""\
     </iframe>
 </body></html>"""
 
+PROBLEMATIC_LINKS = b"""\
+<html><head>
+    <title>Hello</title>
+</head>
+<body>
+    <a href="https://budplaceholder.com/">Placeholder</a>
+    <a href="mailto:test@test.com">test@test.com</a>
+    <a href=https://avatars.githubusercontent.com/apify>Apify avatar/a>
+</body></html>"""
+
 GENERIC_RESPONSE = b"""\
 <html><head>
     <title>Hello</title>