crawlee 1.1.1b11__tar.gz → 1.1.1b13__tar.gz

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/CHANGELOG.md

@@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file.
 - Fix `same-domain` strategy ignoring public suffix  ([#1572](https://github.com/apify/crawlee-python/pull/1572)) ([3d018b2](https://github.com/apify/crawlee-python/commit/3d018b21a28a4bee493829783057188d6106a69b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1571](https://github.com/apify/crawlee-python/issues/1571)
 - Make context helpers work in `FailedRequestHandler` and `ErrorHandler` ([#1570](https://github.com/apify/crawlee-python/pull/1570)) ([b830019](https://github.com/apify/crawlee-python/commit/b830019350830ac33075316061659e2854f7f4a5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1532](https://github.com/apify/crawlee-python/issues/1532)
 - Fix non-ASCII character corruption in `FileSystemStorageClient` on systems without UTF-8 default encoding ([#1580](https://github.com/apify/crawlee-python/pull/1580)) ([f179f86](https://github.com/apify/crawlee-python/commit/f179f8671b0b6af9264450e4fef7e49d1cecd2bd)) by [@Mantisus](https://github.com/Mantisus), closes [#1579](https://github.com/apify/crawlee-python/issues/1579)
+- Respect `<base>` when enqueuing ([#1590](https://github.com/apify/crawlee-python/pull/1590)) ([de517a1](https://github.com/apify/crawlee-python/commit/de517a1629cc29b20568143eb64018f216d4ba33)) by [@Mantisus](https://github.com/Mantisus), closes [#1589](https://github.com/apify/crawlee-python/issues/1589)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.1.1b11
+Version: 1.1.1b13
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.1.1b11"
+version = "1.1.1b13"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -167,9 +167,15 @@ class AbstractHttpCrawler(
             kwargs.setdefault('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+
+            # Get base URL from <base> tag if present
+            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]'))
+            base_url: str = (
+                str(extracted_base_urls[0])
+                if extracted_base_urls
+                else context.request.loaded_url or context.request.url
             )
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -369,9 +369,12 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
-            )
+
+            # Get base URL from <base> tag if present
+            extracted_base_url = await context.page.evaluate('document.baseURI')
+            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/e2e/project_template/test_static_crawlers_templates.py

@@ -71,6 +71,9 @@ async def test_static_crawler_actor_at_apify(
         project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path, package_manager=package_manager
     )
 
+    # Print apify version for debugging purposes in rare cases of CLI failures
+    subprocess.run(['apify', '--version'], check=True)  # noqa: ASYNC221, S607
+
     # Build actor using sequence of cli commands as the user would
     subprocess.run(  # noqa: ASYNC221, S603
         ['apify', 'login', '-t', os.environ['APIFY_TEST_USER_API_TOKEN']],  # noqa: S607

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -58,6 +58,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_cli
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -131,6 +134,9 @@ async def test_enqueue_links_with_transform_request_function(server_url: URL, ht
         str(server_url / 'sub_index'),
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
+        str(server_url / 'base_page'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
     # # all urls added to `enqueue_links` must have a custom header
@@ -164,6 +170,8 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -221,6 +229,7 @@ async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> N
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -61,6 +61,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_cli
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -151,6 +154,9 @@ async def test_enqueue_links_with_transform_request_function(server_url: URL, ht
         str(server_url / 'sub_index'),
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
     # # all urls added to `enqueue_links` must have a custom header
@@ -258,6 +264,8 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -315,6 +323,7 @@ async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> N
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -99,6 +99,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL) -> None:
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -668,6 +671,8 @@ async def test_respect_robots_txt(server_url: URL) -> None:
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -724,6 +729,7 @@ async def test_on_skipped_request(server_url: URL) -> None:
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/unit/server.py

@@ -15,6 +15,7 @@ from uvicorn.server import Server
 from yarl import URL
 
 from tests.unit.server_endpoints import (
+    BASE_INDEX,
     GENERIC_RESPONSE,
     HELLO_WORLD,
     INCAPSULA,
@@ -105,6 +106,7 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
         'page_1': generic_response_endpoint,
         'page_2': generic_response_endpoint,
         'page_3': generic_response_endpoint,
+        'base_page': base_index_endpoint,
         'problematic_links': problematic_links_endpoint,
         'set_cookies': set_cookies,
         'set_complex_cookies': set_complex_cookies,
@@ -431,6 +433,16 @@ async def resource_loading_endpoint(_scope: dict[str, Any], _receive: Receive, s
     )
 
 
+async def base_index_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    """Handle requests for the base index page."""
+    host = f'http://{get_headers_dict(_scope).get("host", "localhost")}'
+    content = BASE_INDEX.format(host=host).encode()
+    await send_html_response(
+        send,
+        content,
+    )
+
+
 class TestServer(Server):
     """A test HTTP server implementation based on Uvicorn Server."""
 

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/tests/unit/server_endpoints.py

@@ -24,6 +24,18 @@ SECONDARY_INDEX = b"""\
 <body>
     <a href="/page_3">Link 3</a>
     <a href="/page_2">Link 4</a>
+    <a href="/base_page">Base Page</a>
+</body></html>"""
+
+BASE_INDEX = """\
+<html><head>
+    <base href="{host}/base_subpath/">
+    <base href="{host}/sub_index/">
+    <title>Hello</title>
+</head>
+<body>
+    <a href="page_5">Link 5</a>
+    <a href="/page_4">Link 6</a>
 </body></html>"""
 
 INCAPSULA = b"""\

{crawlee-1.1.1b11 → crawlee-1.1.1b13}/uv.lock

@@ -706,7 +706,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.1.1b11"
+version = "1.1.1b13"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },