crawlee 1.1.1b12__tar.gz → 1.1.2b1__tar.gz

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/CHANGELOG.md

@@ -3,7 +3,11 @@
 All notable changes to this project will be documented in this file.
 
 <!-- git-cliff-unreleased-start -->
-## 1.1.1 - **not yet released**
+## 1.1.2 - **not yet released**
+
+
+<!-- git-cliff-unreleased-end -->
+## [1.1.1](https://github.com/apify/crawlee-python/releases/tag/v1.1.1) (2025-12-02)
 
 ### 🐛 Bug Fixes
 
@@ -11,9 +15,9 @@ All notable changes to this project will be documented in this file.
 - Fix `same-domain` strategy ignoring public suffix  ([#1572](https://github.com/apify/crawlee-python/pull/1572)) ([3d018b2](https://github.com/apify/crawlee-python/commit/3d018b21a28a4bee493829783057188d6106a69b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1571](https://github.com/apify/crawlee-python/issues/1571)
 - Make context helpers work in `FailedRequestHandler` and `ErrorHandler` ([#1570](https://github.com/apify/crawlee-python/pull/1570)) ([b830019](https://github.com/apify/crawlee-python/commit/b830019350830ac33075316061659e2854f7f4a5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1532](https://github.com/apify/crawlee-python/issues/1532)
 - Fix non-ASCII character corruption in `FileSystemStorageClient` on systems without UTF-8 default encoding ([#1580](https://github.com/apify/crawlee-python/pull/1580)) ([f179f86](https://github.com/apify/crawlee-python/commit/f179f8671b0b6af9264450e4fef7e49d1cecd2bd)) by [@Mantisus](https://github.com/Mantisus), closes [#1579](https://github.com/apify/crawlee-python/issues/1579)
+- Respect `&lt;base&gt;` when enqueuing ([#1590](https://github.com/apify/crawlee-python/pull/1590)) ([de517a1](https://github.com/apify/crawlee-python/commit/de517a1629cc29b20568143eb64018f216d4ba33)) by [@Mantisus](https://github.com/Mantisus), closes [#1589](https://github.com/apify/crawlee-python/issues/1589)
 
 
-<!-- git-cliff-unreleased-end -->
 ## [1.1.0](https://github.com/apify/crawlee-python/releases/tag/v1.1.0) (2025-11-18)
 
 ### 🚀 Features

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.1.1b12
+Version: 1.1.2b1
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -247,7 +247,7 @@ Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
 Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -301,7 +301,7 @@ Requires-Dist: playwright>=1.27.0; extra == 'playwright'
 Provides-Extra: redis
 Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'sql-postgres'
+Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
 Provides-Extra: sql-sqlite
 Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.1.1b12"
+version = "1.1.2b1"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -74,7 +74,7 @@ otel = [
 ]
 sql_postgres = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",
-    "asyncpg>=0.24.0; python_version < '3.14'" # TODO: https://github.com/apify/crawlee-python/issues/1555
+    "asyncpg>=0.24.0"
 ]
 sql_sqlite = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -167,9 +167,15 @@ class AbstractHttpCrawler(
             kwargs.setdefault('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+
+            # Get base URL from <base> tag if present
+            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]'))
+            base_url: str = (
+                str(extracted_base_urls[0])
+                if extracted_base_urls
+                else context.request.loaded_url or context.request.url
             )
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -369,9 +369,12 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
-            )
+
+            # Get base URL from <base> tag if present
+            extracted_base_url = await context.page.evaluate('document.baseURI')
+            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/src/crawlee/storage_clients/_sql/_storage_client.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import warnings
 from datetime import timedelta
 from pathlib import Path
@@ -269,14 +268,6 @@ class SqlStorageClient(StorageClient):
                 'Unsupported database. Supported: sqlite, postgresql. Consider using a different database.'
             )
 
-        # TODO: https://github.com/apify/crawlee-python/issues/1555
-        if 'postgresql' in connection_string and sys.version_info >= (3, 14):
-            raise ValueError(
-                'SqlStorageClient cannot use PostgreSQL with Python 3.14 '
-                'due to asyncpg compatibility limitations. '
-                'Please use Python 3.13 or earlier, or switch to SQLite.'
-            )
-
         self._engine = create_async_engine(
             connection_string,
             future=True,

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -58,6 +58,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_cli
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -131,6 +134,9 @@ async def test_enqueue_links_with_transform_request_function(server_url: URL, ht
         str(server_url / 'sub_index'),
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
+        str(server_url / 'base_page'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
     # # all urls added to `enqueue_links` must have a custom header
@@ -164,6 +170,8 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -221,6 +229,7 @@ async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> N
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -61,6 +61,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_cli
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -151,6 +154,9 @@ async def test_enqueue_links_with_transform_request_function(server_url: URL, ht
         str(server_url / 'sub_index'),
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
     # # all urls added to `enqueue_links` must have a custom header
@@ -258,6 +264,8 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -315,6 +323,7 @@ async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> N
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -99,6 +99,9 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL) -> None:
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -668,6 +671,8 @@ async def test_respect_robots_txt(server_url: URL) -> None:
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
+        str(server_url / 'base_page'),
+        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -724,6 +729,7 @@ async def test_on_skipped_request(server_url: URL) -> None:
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
+        str(server_url / 'page_4'),
     }
 
 

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/tests/unit/server.py

@@ -15,6 +15,7 @@ from uvicorn.server import Server
 from yarl import URL
 
 from tests.unit.server_endpoints import (
+    BASE_INDEX,
     GENERIC_RESPONSE,
     HELLO_WORLD,
     INCAPSULA,
@@ -105,6 +106,7 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None:
         'page_1': generic_response_endpoint,
         'page_2': generic_response_endpoint,
         'page_3': generic_response_endpoint,
+        'base_page': base_index_endpoint,
         'problematic_links': problematic_links_endpoint,
         'set_cookies': set_cookies,
         'set_complex_cookies': set_complex_cookies,
@@ -431,6 +433,16 @@ async def resource_loading_endpoint(_scope: dict[str, Any], _receive: Receive, s
     )
 
 
+async def base_index_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    """Handle requests for the base index page."""
+    host = f'http://{get_headers_dict(_scope).get("host", "localhost")}'
+    content = BASE_INDEX.format(host=host).encode()
+    await send_html_response(
+        send,
+        content,
+    )
+
+
 class TestServer(Server):
     """A test HTTP server implementation based on Uvicorn Server."""
 

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/tests/unit/server_endpoints.py

@@ -24,6 +24,18 @@ SECONDARY_INDEX = b"""\
 <body>
     <a href="/page_3">Link 3</a>
     <a href="/page_2">Link 4</a>
+    <a href="/base_page">Base Page</a>
+</body></html>"""
+
+BASE_INDEX = """\
+<html><head>
+    <base href="{host}/base_subpath/">
+    <base href="{host}/sub_index/">
+    <title>Hello</title>
+</head>
+<body>
+    <a href="page_5">Link 5</a>
+    <a href="/page_4">Link 6</a>
 </body></html>"""
 
 INCAPSULA = b"""\

{crawlee-1.1.1b12 → crawlee-1.1.2b1}/uv.lock

@@ -706,7 +706,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.1.1b12"
+version = "1.1.2b1"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },
@@ -734,7 +734,7 @@ adaptive-crawler = [
 all = [
     { name = "aiosqlite" },
     { name = "apify-fingerprint-datapoints" },
-    { name = "asyncpg", marker = "python_full_version < '3.14'" },
+    { name = "asyncpg" },
     { name = "beautifulsoup4", extra = ["lxml"] },
     { name = "browserforge" },
     { name = "cookiecutter" },
@@ -797,7 +797,7 @@ redis = [
     { name = "redis", extra = ["hiredis"] },
 ]
 sql-postgres = [
-    { name = "asyncpg", marker = "python_full_version < '3.14'" },
+    { name = "asyncpg" },
     { name = "sqlalchemy", extra = ["asyncio"] },
 ]
 sql-sqlite = [
@@ -836,7 +836,7 @@ requires-dist = [
     { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.2" },
     { name = "apify-fingerprint-datapoints", marker = "extra == 'httpx'", specifier = ">=0.0.2" },
     { name = "apify-fingerprint-datapoints", marker = "extra == 'playwright'", specifier = ">=0.0.2" },
-    { name = "asyncpg", marker = "python_full_version < '3.14' and extra == 'sql-postgres'", specifier = ">=0.24.0" },
+    { name = "asyncpg", marker = "extra == 'sql-postgres'", specifier = ">=0.24.0" },
     { name = "beautifulsoup4", extras = ["lxml"], marker = "extra == 'beautifulsoup'", specifier = ">=4.12.0" },
     { name = "browserforge", marker = "extra == 'adaptive-crawler'", specifier = ">=1.2.3" },
     { name = "browserforge", marker = "extra == 'httpx'", specifier = ">=1.2.3" },