crawlee 1.0.5b15__tar.gz → 1.0.5b17__tar.gz

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/.github/workflows/build_and_deploy_docs.yaml

@@ -10,7 +10,7 @@ on:
 
 env:
   NODE_VERSION: 20
-  PYTHON_VERSION: 3.13
+  PYTHON_VERSION: 3.14
 
 jobs:
   build_and_deploy_docs:

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/.github/workflows/release.yaml

@@ -47,13 +47,13 @@ jobs:
     name: Lint check
     uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   type_check:
     name: Type check
     uses: apify/workflows/.github/workflows/python_type_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   unit_tests:
     name: Unit tests
@@ -61,7 +61,7 @@ jobs:
     secrets:
       httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   update_changelog:
     name: Update changelog

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/.github/workflows/run_code_checks.yaml

@@ -21,13 +21,13 @@ jobs:
     name: Lint check
     uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   type_check:
     name: Type check
     uses: apify/workflows/.github/workflows/python_type_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   unit_tests:
     name: Unit tests
@@ -35,7 +35,7 @@ jobs:
     secrets:
       httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   docs_check:
     name: Docs check

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/.github/workflows/templates_e2e_tests.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   NODE_VERSION: 22
-  PYTHON_VERSION: 3.13
+  PYTHON_VERSION: 3.14
 
 jobs:
   end_to_end_tests:

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 
 - Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
 - Add `RedisStorageClient` based on Redis v8.0+ ([#1406](https://github.com/apify/crawlee-python/pull/1406)) ([d08d13d](https://github.com/apify/crawlee-python/commit/d08d13d39203c24ab61fe254b0956d6744db3b5f)) by [@Mantisus](https://github.com/Mantisus)
+- Add support for Python 3.14 ([#1553](https://github.com/apify/crawlee-python/pull/1553)) ([89e9130](https://github.com/apify/crawlee-python/commit/89e9130cabee0fbc974b29c26483b7fa0edf627c)) by [@Mantisus](https://github.com/Mantisus)
 
 ### 🐛 Bug Fixes
 

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b15
+Version: 1.0.5b17
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -223,11 +223,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
-Requires-Dist: impit>=0.6.1
+Requires-Dist: impit>=0.8.0
 Requires-Dist: more-itertools>=10.2.0
 Requires-Dist: protego>=0.5.0
 Requires-Dist: psutil>=6.0.0
@@ -246,7 +247,7 @@ Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
 Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
-Requires-Dist: asyncpg>=0.24.0; extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -300,7 +301,7 @@ Requires-Dist: playwright>=1.27.0; extra == 'playwright'
 Provides-Extra: redis
 Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
-Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
+Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
 Provides-Extra: sql-sqlite
 Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.5b15"
+version = "1.0.5b17"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Libraries",
 ]
 keywords = [
@@ -35,7 +36,7 @@ keywords = [
 dependencies = [
     "cachetools>=5.5.0",
     "colorama>=0.4.0",
-    "impit>=0.6.1",
+    "impit>=0.8.0",
     "more-itertools>=10.2.0",
     "protego>=0.5.0",
     "psutil>=6.0.0",
@@ -73,7 +74,7 @@ otel = [
 ]
 sql_postgres = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",
-    "asyncpg>=0.24.0"
+    "asyncpg>=0.24.0; python_version < '3.14'" # TODO: https://github.com/apify/crawlee-python/issues/1555
 ]
 sql_sqlite = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/src/crawlee/storage_clients/_sql/_storage_client.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import sys
 import warnings
 from datetime import timedelta
 from pathlib import Path
@@ -268,6 +269,14 @@ class SqlStorageClient(StorageClient):
                 'Unsupported database. Supported: sqlite, postgresql. Consider using a different database.'
             )
 
+        # TODO: https://github.com/apify/crawlee-python/issues/1555
+        if 'postgresql' in connection_string and sys.version_info >= (3, 14):
+            raise ValueError(
+                'SqlStorageClient cannot use PostgreSQL with Python 3.14 '
+                'due to asyncpg compatibility limitations. '
+                'Please use Python 3.13 or earlier, or switch to SQLite.'
+            )
+
         self._engine = create_async_engine(
             connection_string,
             future=True,

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/tests/unit/_utils/test_system.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import sys
-from multiprocessing import Barrier, Process, Value, synchronize
+from multiprocessing import get_context, synchronize
 from multiprocessing.shared_memory import SharedMemory
 from typing import TYPE_CHECKING
 
@@ -38,7 +38,9 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
     equal to additional_memory_size_estimate_per_unshared_memory_child where the additional shared memory is exactly
     the same as the unshared memory.
     """
-    estimated_memory_expectation = Value('b', False)  # noqa: FBT003  # Common usage pattern for multiprocessing.Value
+
+    ctx = get_context('fork')
+    estimated_memory_expectation = ctx.Value('b', False)  # noqa: FBT003  # Common usage pattern for multiprocessing.Value
 
     def parent_process() -> None:
         extra_memory_size = 1024 * 1024 * 100  # 100 MB
@@ -70,8 +72,8 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
             *, target: Callable, count: int = 1, use_shared_memory: bool = False
         ) -> float:
             processes = []
-            ready = Barrier(parties=count + 1)
-            measured = Barrier(parties=count + 1)
+            ready = ctx.Barrier(parties=count + 1)
+            measured = ctx.Barrier(parties=count + 1)
             shared_memory: None | SharedMemory = None
             memory_before = get_memory_info().current_size
 
@@ -83,7 +85,7 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
                 extra_args = []
 
             for _ in range(count):
-                p = Process(target=target, args=[ready, measured, *extra_args])
+                p = ctx.Process(target=target, args=[ready, measured, *extra_args])
                 p.start()
                 processes.append(p)
 
@@ -129,7 +131,7 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
                 f'{memory_estimation_difference_ratio=}'
             )
 
-    process = Process(target=parent_process)
+    process = ctx.Process(target=parent_process)
     process.start()
     process.join()
 

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1673,6 +1673,7 @@ def _process_run_crawler(requests: list[str], storage_dir: str) -> StatisticsSta
     return asyncio.run(_run_crawler(requests=requests, storage_dir=storage_dir))
 
 
+@pytest.mark.skip(reason='This test is flaky, see https://github.com/apify/crawlee-python/issues/1560.')
 async def test_crawler_statistics_persistence(tmp_path: Path) -> None:
     """Test that crawler statistics persist and are loaded correctly.
 

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/tests/unit/storage_clients/_redis/test_redis_rq_client.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import sys
 from typing import TYPE_CHECKING
 
 import pytest
@@ -25,6 +26,10 @@ async def rq_client(
     suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[RedisRequestQueueClient, None]:
     """A fixture for a Redis RQ client."""
+    # TODO: https://github.com/apify/crawlee-python/issues/1554
+    if request.param == 'bloom' and sys.platform == 'win32' and sys.version_info >= (3, 14):
+        pytest.skip('Bloom filters not supported on Windows with Python 3.14 and fakeredis')
+
     client = await RedisStorageClient(redis=redis_client, queue_dedup_strategy=request.param).create_rq_client(
         name='test_request_queue'
     )

{crawlee-1.0.5b15 → crawlee-1.0.5b17}/uv.lock

@@ -705,7 +705,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.5b15"
+version = "1.0.5b17"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },
@@ -733,7 +733,7 @@ adaptive-crawler = [
 all = [
     { name = "aiosqlite" },
     { name = "apify-fingerprint-datapoints" },
-    { name = "asyncpg" },
+    { name = "asyncpg", marker = "python_full_version < '3.14'" },
     { name = "beautifulsoup4", extra = ["lxml"] },
     { name = "browserforge" },
     { name = "cookiecutter" },
@@ -796,7 +796,7 @@ redis = [
     { name = "redis", extra = ["hiredis"] },
 ]
 sql-postgres = [
-    { name = "asyncpg" },
+    { name = "asyncpg", marker = "python_full_version < '3.14'" },
     { name = "sqlalchemy", extra = ["asyncio"] },
 ]
 sql-sqlite = [
@@ -835,7 +835,7 @@ requires-dist = [
     { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.2" },
     { name = "apify-fingerprint-datapoints", marker = "extra == 'httpx'", specifier = ">=0.0.2" },
     { name = "apify-fingerprint-datapoints", marker = "extra == 'playwright'", specifier = ">=0.0.2" },
-    { name = "asyncpg", marker = "extra == 'sql-postgres'", specifier = ">=0.24.0" },
+    { name = "asyncpg", marker = "python_full_version < '3.14' and extra == 'sql-postgres'", specifier = ">=0.24.0" },
     { name = "beautifulsoup4", extras = ["lxml"], marker = "extra == 'beautifulsoup'", specifier = ">=4.12.0" },
     { name = "browserforge", marker = "extra == 'adaptive-crawler'", specifier = ">=1.2.3" },
     { name = "browserforge", marker = "extra == 'httpx'", specifier = ">=1.2.3" },
@@ -847,7 +847,7 @@ requires-dist = [
     { name = "curl-cffi", marker = "extra == 'curl-impersonate'", specifier = ">=0.9.0" },
     { name = "html5lib", marker = "extra == 'beautifulsoup'", specifier = ">=1.0" },
     { name = "httpx", extras = ["brotli", "http2", "zstd"], marker = "extra == 'httpx'", specifier = ">=0.27.0" },
-    { name = "impit", specifier = ">=0.6.1" },
+    { name = "impit", specifier = ">=0.8.0" },
     { name = "inquirer", marker = "extra == 'cli'", specifier = ">=3.3.0" },
     { name = "jaro-winkler", marker = "extra == 'adaptive-crawler'", specifier = ">=2.0.3" },
     { name = "more-itertools", specifier = ">=10.2.0" },