crawlee 1.1.1b12__py3-none-any.whl → 1.1.2b1__py3-none-any.whl

crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -167,9 +167,15 @@ class AbstractHttpCrawler(
             kwargs.setdefault('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+
+            # Get base URL from <base> tag if present
+            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]'))
+            base_url: str = (
+                str(extracted_base_urls[0])
+                if extracted_base_urls
+                else context.request.loaded_url or context.request.url
             )
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -369,9 +369,12 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
-            )
+
+            # Get base URL from <base> tag if present
+            extracted_base_url = await context.page.evaluate('document.baseURI')
+            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

crawlee/storage_clients/_sql/_storage_client.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import warnings
 from datetime import timedelta
 from pathlib import Path
@@ -269,14 +268,6 @@ class SqlStorageClient(StorageClient):
                 'Unsupported database. Supported: sqlite, postgresql. Consider using a different database.'
             )
 
-        # TODO: https://github.com/apify/crawlee-python/issues/1555
-        if 'postgresql' in connection_string and sys.version_info >= (3, 14):
-            raise ValueError(
-                'SqlStorageClient cannot use PostgreSQL with Python 3.14 '
-                'due to asyncpg compatibility limitations. '
-                'Please use Python 3.13 or earlier, or switch to SQLite.'
-            )
-
         self._engine = create_async_engine(
             connection_string,
             future=True,

{crawlee-1.1.1b12.dist-info → crawlee-1.1.2b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.1.1b12
+Version: 1.1.2b1
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -247,7 +247,7 @@ Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
 Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -301,7 +301,7 @@ Requires-Dist: playwright>=1.27.0; extra == 'playwright'
 Provides-Extra: redis
 Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'sql-postgres'
+Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
 Provides-Extra: sql-sqlite
 Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'

{crawlee-1.1.1b12.dist-info → crawlee-1.1.2b1.dist-info}/RECORD

@@ -53,7 +53,7 @@ crawlee/crawlers/__init__.py,sha256=9VmFahav3rjE-2Bxa5PAhBgkYXP0k5SSAEpdG2xMZ7c,
 crawlee/crawlers/_types.py,sha256=xbGTJQirgz5wUbfr12afMR4q-_5AWP7ngF2e8K5P8l0,355
 crawlee/crawlers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlee/crawlers/_abstract_http/__init__.py,sha256=QCjn8x7jpo8FwEeSRw10TVj_0La2v9mLEiQWdk2RoTw,273
-crawlee/crawlers/_abstract_http/_abstract_http_crawler.py,sha256=DEiErZi7j2FHMgyVELPy09GyHo5Gx4UDpuiN6D3sGNk,11553
+crawlee/crawlers/_abstract_http/_abstract_http_crawler.py,sha256=wFErk_Mg02vlgX_jV5GohmZs2qVZt3wy1yBKrU0lbAc,11829
 crawlee/crawlers/_abstract_http/_abstract_http_parser.py,sha256=Y5o_hiW_0mQAte5GFqkUxscwKEFpWrBYRsLKP1cfBwE,3521
 crawlee/crawlers/_abstract_http/_http_crawling_context.py,sha256=Rno_uJ8ivmyRxFQv2MyY_z9B5WPHSEd5MAPz31_1ZIo,2179
 crawlee/crawlers/_abstract_http/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -85,7 +85,7 @@ crawlee/crawlers/_parsel/_parsel_crawling_context.py,sha256=sZB26RcRLjSoD15myEOM
 crawlee/crawlers/_parsel/_parsel_parser.py,sha256=yWBfuXUHMriK4DRnyrXTQoGeqX5WV9bOEkBp_g0YCvQ,1540
 crawlee/crawlers/_parsel/_utils.py,sha256=MbRwx-cdjlq1zLzFYf64M3spOGQ6yxum4FvP0sdqA_Q,2693
 crawlee/crawlers/_playwright/__init__.py,sha256=6Cahe6VEF82o8CYiP8Cmp58Cmb6Rb8uMeyy7wnwe5ms,837
-crawlee/crawlers/_playwright/_playwright_crawler.py,sha256=53iytj5LZHw19QOYqYNlZL4ApPlhbWn9Ds-DPTKANhQ,24158
+crawlee/crawlers/_playwright/_playwright_crawler.py,sha256=2ONpyYRjX31YmgFhx2pVkt-SBXZCS-HawAtsG99fwYc,24324
 crawlee/crawlers/_playwright/_playwright_crawling_context.py,sha256=Oi0tMBXHaEDlFjqG01DzgB7Ck52bjVjz-X__eMioxas,1249
 crawlee/crawlers/_playwright/_playwright_http_client.py,sha256=Nfm69dqX85k68jN1p3ljZWbn8egqDWPIPRykXyXsoQs,3977
 crawlee/crawlers/_playwright/_playwright_pre_nav_crawling_context.py,sha256=fEI2laWhmJdWiGoMF5JBLBsim9NtENfagZt6FFd2Rgo,1387
@@ -189,7 +189,7 @@ crawlee/storage_clients/_sql/_dataset_client.py,sha256=tiJVvOPZgc7cy4kGfWnun-g2T
 crawlee/storage_clients/_sql/_db_models.py,sha256=KzA-R_L6zv9gqQg7B27mF-fERNJuMUEnewV9iofmTnI,9812
 crawlee/storage_clients/_sql/_key_value_store_client.py,sha256=LnVLWhOjo4LdvtCac4fwuf__DgEQjlqSxz8KkjY3Qx4,11311
 crawlee/storage_clients/_sql/_request_queue_client.py,sha256=OlvAOwEoYY5f4NO7BdhLFRT_i_E3YzJDb_ptKKK2huY,29478
-crawlee/storage_clients/_sql/_storage_client.py,sha256=okONooE_1Z5hxhJaXl7jEDXEryKqXlnFx6F1EuxyP2s,11317
+crawlee/storage_clients/_sql/_storage_client.py,sha256=ITtMpwfotIW4SZjO4rycB5wfMKaqTAJgMvzcUZxckrk,10905
 crawlee/storage_clients/_sql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlee/storages/__init__.py,sha256=wc2eioyCKAAYrg4N7cshpjC-UbE23OzGar9nK_kteSY,186
 crawlee/storages/_base.py,sha256=zUOcMJTg8MAzq-m9X1NJcWncCfxzI5mb5MyY35WAkMk,2310
@@ -199,8 +199,8 @@ crawlee/storages/_request_queue.py,sha256=bjBOGbpMaGUsqJPVB-JD2VShziPAYMI-GvWKKp
 crawlee/storages/_storage_instance_manager.py,sha256=72n0YlPwNpSQDJSPf4TxnI2GvIK6L-ZiTmHRbFcoVU0,8164
 crawlee/storages/_utils.py,sha256=Yz-5tEBYKYCFJemYT29--uGJqoJLApLDLgPcsnbifRw,439
 crawlee/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlee-1.1.1b12.dist-info/METADATA,sha256=Mnw0BlsITcdcKnq5zMqsz-BRyp5CEDwQvgRNjHsr1f4,29533
-crawlee-1.1.1b12.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-crawlee-1.1.1b12.dist-info/entry_points.txt,sha256=1p65X3dA-cYvzjtlxLL6Kn1wpY-3uEDVqJLp53uNPeo,45
-crawlee-1.1.1b12.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-crawlee-1.1.1b12.dist-info/RECORD,,
+crawlee-1.1.2b1.dist-info/METADATA,sha256=VcyGEp0Jf5SfylsXe_2tpxo5dH8_wRM9eqUqgynRRJk,29472
+crawlee-1.1.2b1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+crawlee-1.1.2b1.dist-info/entry_points.txt,sha256=1p65X3dA-cYvzjtlxLL6Kn1wpY-3uEDVqJLp53uNPeo,45
+crawlee-1.1.2b1.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+crawlee-1.1.2b1.dist-info/RECORD,,