crawlee 1.0.5b3__tar.gz → 1.0.5b16__tar.gz

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/.github/workflows/build_and_deploy_docs.yaml

@@ -10,7 +10,7 @@ on:
 
 env:
   NODE_VERSION: 20
-  PYTHON_VERSION: 3.13
+  PYTHON_VERSION: 3.14
 
 jobs:
   build_and_deploy_docs:

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/.github/workflows/release.yaml

@@ -47,13 +47,13 @@ jobs:
     name: Lint check
     uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   type_check:
     name: Type check
     uses: apify/workflows/.github/workflows/python_type_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   unit_tests:
     name: Unit tests
@@ -61,7 +61,7 @@ jobs:
     secrets:
       httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   update_changelog:
     name: Update changelog

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/.github/workflows/run_code_checks.yaml

@@ -21,13 +21,13 @@ jobs:
     name: Lint check
     uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   type_check:
     name: Type check
     uses: apify/workflows/.github/workflows/python_type_check.yaml@main
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   unit_tests:
     name: Unit tests
@@ -35,8 +35,9 @@ jobs:
     secrets:
       httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org'}}
     with:
-      python-versions: '["3.10", "3.11", "3.12", "3.13"]'
+      python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
 
   docs_check:
     name: Docs check
     uses: apify/workflows/.github/workflows/python_docs_check.yaml@main
+    secrets: inherit

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/.github/workflows/templates_e2e_tests.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   NODE_VERSION: 22
-  PYTHON_VERSION: 3.13
+  PYTHON_VERSION: 3.14
 
 jobs:
   end_to_end_tests:

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/CHANGELOG.md

@@ -8,10 +8,14 @@ All notable changes to this project will be documented in this file.
 ### 🚀 Features
 
 - Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
+- Add `RedisStorageClient` based on Redis v8.0+ ([#1406](https://github.com/apify/crawlee-python/pull/1406)) ([d08d13d](https://github.com/apify/crawlee-python/commit/d08d13d39203c24ab61fe254b0956d6744db3b5f)) by [@Mantisus](https://github.com/Mantisus)
+- Add support for Python 3.14 ([#1553](https://github.com/apify/crawlee-python/pull/1553)) ([89e9130](https://github.com/apify/crawlee-python/commit/89e9130cabee0fbc974b29c26483b7fa0edf627c)) by [@Mantisus](https://github.com/Mantisus)
 
 ### 🐛 Bug Fixes
 
 - Improve indexing of the `request_queue_records` table for `SqlRequestQueueClient` ([#1527](https://github.com/apify/crawlee-python/pull/1527)) ([6509534](https://github.com/apify/crawlee-python/commit/65095346a9d8b703b10c91e0510154c3c48a4176)) by [@Mantisus](https://github.com/Mantisus), closes [#1526](https://github.com/apify/crawlee-python/issues/1526)
+- Improve error handling for `RobotsTxtFile.load` ([#1524](https://github.com/apify/crawlee-python/pull/1524)) ([596a311](https://github.com/apify/crawlee-python/commit/596a31184914a254b3e7a81fd2f48ea8eda7db49)) by [@Mantisus](https://github.com/Mantisus)
+- Fix `crawler_runtime` not being updated during run and only in the end ([#1540](https://github.com/apify/crawlee-python/pull/1540)) ([0d6c3f6](https://github.com/apify/crawlee-python/commit/0d6c3f6d3337ddb6cab4873747c28cf95605d550)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1541](https://github.com/apify/crawlee-python/issues/1541)
 
 
 <!-- git-cliff-unreleased-end -->
@@ -281,7 +285,7 @@ All notable changes to this project will be documented in this file.
 
 ### 🐛 Bug Fixes
 
-- Fix session managment with retire ([#947](https://github.com/apify/crawlee-python/pull/947)) ([caee03f](https://github.com/apify/crawlee-python/commit/caee03fe3a43cc1d7a8d3f9e19b42df1bdb1c0aa)) by [@Mantisus](https://github.com/Mantisus)
+- Fix session management with retire ([#947](https://github.com/apify/crawlee-python/pull/947)) ([caee03f](https://github.com/apify/crawlee-python/commit/caee03fe3a43cc1d7a8d3f9e19b42df1bdb1c0aa)) by [@Mantisus](https://github.com/Mantisus)
 - Fix templates - poetry-plugin-export version and camoufox template name ([#952](https://github.com/apify/crawlee-python/pull/952)) ([7addea6](https://github.com/apify/crawlee-python/commit/7addea6605359cceba208e16ec9131724bdb3e9b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#951](https://github.com/apify/crawlee-python/issues/951)
 - Fix convert relative link to absolute in `enqueue_links` for response with redirect ([#956](https://github.com/apify/crawlee-python/pull/956)) ([694102e](https://github.com/apify/crawlee-python/commit/694102e163bb9021a4830d2545d153f6f8f3de90)) by [@Mantisus](https://github.com/Mantisus), closes [#955](https://github.com/apify/crawlee-python/issues/955)
 - Fix `CurlImpersonateHttpClient` cookies handler ([#946](https://github.com/apify/crawlee-python/pull/946)) ([ed415c4](https://github.com/apify/crawlee-python/commit/ed415c433da2a40b0ee62534f0730d0737e991b8)) by [@Mantisus](https://github.com/Mantisus)

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/CONTRIBUTING.md

@@ -103,7 +103,7 @@ make run-docs
 Publishing new versions to [PyPI](https://pypi.org/project/crawlee) is automated through GitHub Actions.
 
 - **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
-- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overriden using the `custom` release type.
+- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
 
 ### Publishing to PyPI manually
 

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/Makefile

@@ -38,7 +38,7 @@ unit-tests-cov:
 	uv run pytest --numprocesses=auto -vv --cov=src/crawlee --cov-append --cov-report=html tests/unit -m "not run_alone"
 
 e2e-templates-tests $(args):
-	uv run pytest --numprocesses=$(E2E_TESTS_CONCURRENCY) -vv tests/e2e/project_template "$(args)"
+	uv run pytest --numprocesses=$(E2E_TESTS_CONCURRENCY) -vv tests/e2e/project_template "$(args)" --timeout=600
 
 format:
 	uv run ruff check --fix

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b3
+Version: 1.0.5b16
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -223,15 +223,16 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
-Requires-Dist: impit>=0.6.1
+Requires-Dist: impit>=0.8.0
 Requires-Dist: more-itertools>=10.2.0
 Requires-Dist: protego>=0.5.0
 Requires-Dist: psutil>=6.0.0
-Requires-Dist: pydantic-settings!=2.7.0,!=2.7.1,!=2.8.0,>=2.2.0
+Requires-Dist: pydantic-settings>=2.12.0
 Requires-Dist: pydantic>=2.11.0
 Requires-Dist: pyee>=9.0.0
 Requires-Dist: tldextract>=5.1.0
@@ -246,7 +247,7 @@ Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
 Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
-Requires-Dist: asyncpg>=0.24.0; extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -263,6 +264,7 @@ Requires-Dist: opentelemetry-sdk>=1.34.1; extra == 'all'
 Requires-Dist: opentelemetry-semantic-conventions>=0.54; extra == 'all'
 Requires-Dist: parsel>=1.10.0; extra == 'all'
 Requires-Dist: playwright>=1.27.0; extra == 'all'
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'all'
 Requires-Dist: rich>=13.9.0; extra == 'all'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'all'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'all'
@@ -296,8 +298,10 @@ Provides-Extra: playwright
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'playwright'
 Requires-Dist: browserforge>=1.2.3; extra == 'playwright'
 Requires-Dist: playwright>=1.27.0; extra == 'playwright'
+Provides-Extra: redis
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
-Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
+Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
 Provides-Extra: sql-sqlite
 Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/docs/deployment/apify_platform.mdx

@@ -99,7 +99,7 @@ apify run
 For running Crawlee code as an Actor on [Apify platform](https://apify.com/actors) you need to wrap the body of the main function of your crawler with `async with Actor`.
 
 :::info NOTE
-Adding `async with Actor` is the only important thing needed to run it on Apify platform as an Actor. It is needed to initialize your Actor (e.g. to set the correct storage implementation) and to correctly handle exitting the process.
+Adding `async with Actor` is the only important thing needed to run it on Apify platform as an Actor. It is needed to initialize your Actor (e.g. to set the correct storage implementation) and to correctly handle exiting the process.
 :::
 
 Let's look at the `BeautifulSoupCrawler` example from the [Quick start](../quick-start) guide:

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/docs/examples/playwright_crawler_with_fingerprint_generator.mdx

@@ -1,5 +1,5 @@
 ---
-id: playwright-crawler-with-fingeprint-generator
+id: playwright-crawler-with-fingerprint-generator
 title: Playwright crawler with fingerprint generator
 ---
 

crawlee-1.0.5b16/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py

@@ -0,0 +1,10 @@
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import RedisStorageClient
+
+# Create a new instance of storage client using connection string.
+# 'redis://localhost:6379' is the just placeholder, replace it with your actual
+# connection string.
+storage_client = RedisStorageClient(connection_string='redis://localhost:6379')
+
+# And pass it to the crawler.
+crawler = ParselCrawler(storage_client=storage_client)

crawlee-1.0.5b16/docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py

@@ -0,0 +1,27 @@
+from redis.asyncio import Redis
+
+from crawlee.configuration import Configuration
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import RedisStorageClient
+
+# Create a new instance of storage client using a Redis client with custom settings.
+# Replace host and port with your actual Redis server configuration.
+# Other Redis client settings can be adjusted as needed.
+storage_client = RedisStorageClient(
+    redis=Redis(
+        host='localhost',
+        port=6379,
+        retry_on_timeout=True,
+        socket_keepalive=True,
+        socket_connect_timeout=10,
+    )
+)
+
+# Create a configuration with custom settings.
+configuration = Configuration(purge_on_start=False)
+
+# And pass them to the crawler.
+crawler = ParselCrawler(
+    storage_client=storage_client,
+    configuration=configuration,
+)

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/docs/guides/storage_clients.mdx

@@ -17,6 +17,8 @@ import CustomStorageClientExample from '!!raw-loader!roa-loader!./code_examples/
 import RegisteringStorageClientsExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/registering_storage_clients_example.py';
 import SQLStorageClientBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/sql_storage_client_basic_example.py';
 import SQLStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/sql_storage_client_configuration_example.py';
+import RedisStorageClientBasicExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_basic_example.py';
+import RedisStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_configuration_example.py';
 
 Storage clients provide a unified interface for interacting with <ApiLink to="class/Dataset">`Dataset`</ApiLink>, <ApiLink to="class/KeyValueStore">`KeyValueStore`</ApiLink>, and <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>, regardless of the underlying implementation. They handle operations like creating, reading, updating, and deleting storage instances, as well as managing data persistence and cleanup. This abstraction makes it easy to switch between different environments, such as local development and cloud production setups.
 
@@ -26,7 +28,8 @@ Crawlee provides three main storage client implementations:
 
 - <ApiLink to="class/FileSystemStorageClient">`FileSystemStorageClient`</ApiLink> - Provides persistent file system storage with in-memory caching.
 - <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> - Stores data in memory with no persistence.
-- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> – Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: 'crawlee[sql_sqlite]' for SQLite or 'crawlee[sql_postgres]' for PostgreSQL.
+- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> - Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: `crawlee[sql_sqlite]` for SQLite or `crawlee[sql_postgres]` for PostgreSQL.
+- <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> - Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`.
 - [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com), implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python).
 
 ```mermaid
@@ -56,6 +59,8 @@ class MemoryStorageClient
 
 class SqlStorageClient
 
+class RedisStorageClient
+
 class ApifyStorageClient
 
 %% ========================
@@ -65,6 +70,7 @@ class ApifyStorageClient
 StorageClient --|> FileSystemStorageClient
 StorageClient --|> MemoryStorageClient
 StorageClient --|> SqlStorageClient
+StorageClient --|> RedisStorageClient
 StorageClient --|> ApifyStorageClient
 ```
 
@@ -304,8 +310,8 @@ Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageCl
 
 Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> can be set via constructor arguments:
 
-- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) – SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
-- **`engine`** – Pre-configured SQLAlchemy AsyncEngine (optional).
+- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) - SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
+- **`engine`** - Pre-configured SQLAlchemy AsyncEngine (optional).
 
 For advanced scenarios, you can configure <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> with a custom SQLAlchemy engine and additional options via the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class. This is useful, for example, when connecting to an external PostgreSQL database or customizing connection pooling.
 
@@ -313,6 +319,172 @@ For advanced scenarios, you can configure <ApiLink to="class/SqlStorageClient">`
     {SQLStorageClientConfigurationExample}
 </CodeBlock>
 
+### Redis storage client
+
+:::warning Experimental feature
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> is experimental. Its API and behavior may change in future releases.
+:::
+
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> provides persistent storage using [Redis](https://redis.io/) database. It supports concurrent access from multiple independent clients or processes and uses Redis native data structures for efficient operations.
+
+:::note dependencies
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> is not included in the core Crawlee package.
+To use it, you need to install Crawlee with the Redis extra dependency:
+
+<code>pip install 'crawlee[redis]'</code>
+
+Additionally, Redis version 8.0 or higher is required.
+:::
+
+:::note Redis persistence
+Data persistence in Redis depends on your [database configuration](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/).
+:::
+
+The client requires either a Redis connection string or a pre-configured Redis client instance. Use a pre-configured client when you need custom Redis settings such as connection pooling, timeouts, or SSL/TLS encryption.
+
+<CodeBlock className="language-python" language="python">
+    {RedisStorageClientBasicExample}
+</CodeBlock>
+
+Data is organized using Redis key patterns. Below are the main data structures used for each storage type:
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Client
+%% ========================
+
+class RedisDatasetClient {
+    <<Dataset>>
+}
+
+%% ========================
+%% Dataset Keys
+%% ========================
+
+class DatasetKeys {
+    datasets:[name]:items - JSON Array
+    datasets:[name]:metadata - JSON Object
+}
+
+class DatasetsIndexes {
+    datasets:id_to_name - Hash
+    datasets:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisDatasetClient --> DatasetKeys
+RedisDatasetClient --> DatasetsIndexes
+```
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Clients
+%% ========================
+
+class RedisKeyValueStoreClient {
+    <<Key-value store>>
+}
+
+%% ========================
+%% Key-Value Store Keys
+%% ========================
+
+class KeyValueStoreKeys {
+    key_value_stores:[name]:items - Hash
+    key_value_stores:[name]:metadata_items - Hash
+    key_value_stores:[name]:metadata - JSON Object
+}
+
+class KeyValueStoresIndexes {
+    key_value_stores:id_to_name - Hash
+    key_value_stores:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisKeyValueStoreClient --> KeyValueStoreKeys
+RedisKeyValueStoreClient --> KeyValueStoresIndexes
+```
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Clients
+%% ========================
+
+class RedisRequestQueueClient {
+    <<Request queue>>
+}
+
+%% ========================
+%% Request Queue Keys
+%% ========================
+
+class RequestQueueKeys{
+    request_queues:[name]:queue - List
+    request_queues:[name]:data - Hash
+    request_queues:[name]:in_progress - Hash
+    request_queues:[name]:added_bloom_filter - Bloom Filter | bloom queue_dedup_strategy
+    request_queues:[name]:handled_bloom_filter - Bloom Filter | bloom queue_dedup_strategy
+    request_queues:[name]:pending_set - Set | default queue_dedup_strategy
+    request_queues:[name]:handled_set - Set | default queue_dedup_strategy
+    request_queues:[name]:metadata - JSON Object
+}
+
+class RequestQueuesIndexes {
+    request_queues:id_to_name - Hash
+    request_queues:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisRequestQueueClient --> RequestQueueKeys
+RedisRequestQueueClient --> RequestQueuesIndexes
+```
+
+Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set through environment variables or the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class:
+
+- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start.
+
+Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set via constructor arguments:
+
+- **`connection_string`** - Redis connection string, e.g. `redis://localhost:6379/0`.
+- **`redis`** - Pre-configured Redis client instance (optional).
+
+<CodeBlock className="language-python" language="python">
+    {RedisStorageClientConfigurationExample}
+</CodeBlock>
+
 ## Creating a custom storage client
 
 A storage client consists of two parts: the storage client factory and individual storage type clients. The <ApiLink to="class/StorageClient">`StorageClient`</ApiLink> acts as a factory that creates specific clients (<ApiLink to="class/DatasetClient">`DatasetClient`</ApiLink>, <ApiLink to="class/KeyValueStoreClient">`KeyValueStoreClient`</ApiLink>, <ApiLink to="class/RequestQueueClient">`RequestQueueClient`</ApiLink>) where the actual storage logic is implemented.

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/docs/guides/trace_and_monitor_crawlers.mdx

@@ -45,7 +45,7 @@ You can use different tools to consume the OpenTelemetry data that might better
 
 ## Customize the instrumentation
 
-You can customize the <ApiLink to="class/CrawlerInstrumentor">`CrawlerInstrumentor`</ApiLink>. Depending on the arguments used during its initialization, the instrumentation will be applied to different parts ot the Crawlee code. By default, it instruments some functions that can give quite a good picture of each individual request handling. To turn this default instrumentation off, you can pass `request_handling_instrumentation=False` during initialization. You can also extend instrumentation by passing `instrument_classes=[...]` initialization argument that contains classes you want to be auto-instrumented. All their public methods will be automatically instrumented. Bear in mind that instrumentation has some runtime costs as well. The more instrumentation is used, the more overhead it will add to the crawler execution.
+You can customize the <ApiLink to="class/CrawlerInstrumentor">`CrawlerInstrumentor`</ApiLink>. Depending on the arguments used during its initialization, the instrumentation will be applied to different parts of the Crawlee code. By default, it instruments some functions that can give quite a good picture of each individual request handling. To turn this default instrumentation off, you can pass `request_handling_instrumentation=False` during initialization. You can also extend instrumentation by passing `instrument_classes=[...]` initialization argument that contains classes you want to be auto-instrumented. All their public methods will be automatically instrumented. Bear in mind that instrumentation has some runtime costs as well. The more instrumentation is used, the more overhead it will add to the crawler execution.
 
 You can also create your instrumentation by selecting only the methods you want to instrument. For more details, see the <ApiLink to="class/CrawlerInstrumentor">`CrawlerInstrumentor`</ApiLink> source code and the [Python documentation for OpenTelemetry](https://opentelemetry.io/docs/languages/python/).
 

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/docs/pyproject.toml

@@ -1,4 +1,4 @@
-# Line lenght different from the rest of the code to make sure that the example codes visualised on the generated
+# Line length different from the rest of the code to make sure that the example codes visualised on the generated
 # documentation webpages are shown without vertical slider to make them more readable.
 
 [tool.ruff]

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.5b3"
+version = "1.0.5b16"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Libraries",
 ]
 keywords = [
@@ -35,11 +36,11 @@ keywords = [
 dependencies = [
     "cachetools>=5.5.0",
     "colorama>=0.4.0",
-    "impit>=0.6.1",
+    "impit>=0.8.0",
     "more-itertools>=10.2.0",
     "protego>=0.5.0",
     "psutil>=6.0.0",
-    "pydantic-settings>=2.2.0,!=2.7.0,!=2.7.1,!=2.8.0",
+    "pydantic-settings>=2.12.0",
     "pydantic>=2.11.0",
     "pyee>=9.0.0",
     "tldextract>=5.1.0",
@@ -48,7 +49,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres]"]
+all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres,redis]"]
 adaptive-crawler = [
     "jaro-winkler>=2.0.3",
     "playwright>=1.27.0",
@@ -73,12 +74,13 @@ otel = [
 ]
 sql_postgres = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",
-    "asyncpg>=0.24.0"
+    "asyncpg>=0.24.0; python_version < '3.14'" # TODO: https://github.com/apify/crawlee-python/issues/1555
 ]
 sql_sqlite = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",
     "aiosqlite>=0.21.0",
 ]
+redis = ["redis[hiredis] >= 7.0.0"]
 
 [project.scripts]
 crawlee = "crawlee._cli:cli"
@@ -96,24 +98,25 @@ crawlee = "crawlee._cli:cli"
 [dependency-groups]
 dev = [
     "apify_client", # For e2e tests.
-    "build~=1.3.0", # For e2e tests.
-    "dycw-pytest-only~=2.1.0",
-    "mypy~=1.18.1",
-    "pre-commit~=4.3.0",
-    "proxy-py~=2.4.0",
-    "pydoc-markdown~=4.8.0",
-    "pytest-asyncio~=1.2.0",
-    "pytest-cov~=7.0.0",
-    "pytest-timeout~=2.4.0",
-    "pytest-xdist~=3.8.0",
-    "pytest~=8.4.0",
+    "build<2.0.0", # For e2e tests.
+    "dycw-pytest-only<3.0.0",
+    "fakeredis[probabilistic,json,lua]<3.0.0",
+    "mypy~=1.18.0",
+    "pre-commit<5.0.0",
+    "proxy-py<3.0.0",
+    "pydoc-markdown<5.0.0",
+    "pytest-asyncio<2.0.0",
+    "pytest-cov<8.0.0",
+    "pytest-timeout<3.0.0",
+    "pytest-xdist<4.0.0",
+    "pytest<9.0.0",
     "ruff~=0.14.0",
     "setuptools", # setuptools are used by pytest, but not explicitly required
-    "types-beautifulsoup4~=4.12.0.20240229",
-    "types-cachetools~=6.2.0.20250827",
-    "types-colorama~=0.4.15.20240106",
-    "types-psutil~=7.0.0.20250218",
-    "types-python-dateutil~=2.9.0.20240316",
+    "types-beautifulsoup4<5.0.0",
+    "types-cachetools<7.0.0",
+    "types-colorama<1.0.0",
+    "types-psutil<8.0.0",
+    "types-python-dateutil<3.0.0",
     "uvicorn[standard]~=0.35.0", # https://github.com/apify/crawlee-python/issues/1441
 ]
 

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/src/crawlee/_utils/robots.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from protego import Protego
@@ -15,6 +16,9 @@ if TYPE_CHECKING:
     from crawlee.proxy_configuration import ProxyInfo
 
 
+logger = getLogger(__name__)
+
+
 class RobotsTxtFile:
     def __init__(
         self, url: str, robots: Protego, http_client: HttpClient | None = None, proxy_info: ProxyInfo | None = None
@@ -56,12 +60,20 @@ class RobotsTxtFile:
             http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file.
             proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
         """
-        response = await http_client.send_request(url, proxy_info=proxy_info)
-        body = (
-            b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else await response.read()
-        )
+        try:
+            response = await http_client.send_request(url, proxy_info=proxy_info)
+
+            body = (
+                b'User-agent: *\nAllow: /'
+                if is_status_code_client_error(response.status_code)
+                else await response.read()
+            )
+            robots = Protego.parse(body.decode('utf-8'))
+
+        except Exception as e:
+            logger.warning(f'Failed to fetch from robots.txt from "{url}" with error: "{e}"')
 
-        robots = Protego.parse(body.decode('utf-8'))
+            robots = Protego.parse('User-agent: *\nAllow: /')
 
         return cls(url, robots, http_client=http_client, proxy_info=proxy_info)
 

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/src/crawlee/configuration.py

@@ -28,7 +28,9 @@ class Configuration(BaseSettings):
     Settings can also be configured via environment variables, prefixed with `CRAWLEE_`.
     """
 
-    model_config = SettingsConfigDict(validate_by_name=True, validate_by_alias=True)
+    # TODO: https://github.com/pydantic/pydantic-settings/issues/706
+    # Use `SettingsConfigDict(validate_by_name=True, validate_by_alias=True)` when issue is resolved.
+    model_config = SettingsConfigDict(populate_by_name=True)
 
     internal_timeout: Annotated[timedelta | None, Field(alias='crawlee_internal_timeout')] = None
     """Timeout for the internal asynchronous operations."""

{crawlee-1.0.5b3 → crawlee-1.0.5b16}/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -71,7 +71,6 @@ class _NonPersistentStatistics(Statistics):
     async def __aenter__(self) -> Self:
         self._active = True
         await self._state.initialize()
-        self._after_initialize()
         return self
 
     async def __aexit__(