crawlee 1.0.1b7__tar.gz → 1.0.5b3__tar.gz

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/.github/workflows/build_and_deploy_docs.yaml

@@ -30,7 +30,7 @@ jobs:
           ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
 
       - name: Set up Node
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ env.NODE_VERSION }}
 
@@ -40,7 +40,7 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Set up uv package manager
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/.github/workflows/templates_e2e_tests.yaml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Setup node
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ env.NODE_VERSION }}
 
@@ -44,7 +44,7 @@ jobs:
         run: pipx install poetry
 
       - name: Set up uv package manager
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/.gitignore

@@ -30,6 +30,7 @@ htmlcov
 # IDE, editors
 .vscode
 .idea
+*~
 .DS_Store
 .nvim.lua
 Session.vim

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/CHANGELOG.md

@@ -3,16 +3,61 @@
 All notable changes to this project will be documented in this file.
 
 <!-- git-cliff-unreleased-start -->
-## 1.0.1 - **not yet released**
+## 1.0.5 - **not yet released**
+
+### 🚀 Features
+
+- Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
 
 ### 🐛 Bug Fixes
 
-- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+- Improve indexing of the `request_queue_records` table for `SqlRequestQueueClient` ([#1527](https://github.com/apify/crawlee-python/pull/1527)) ([6509534](https://github.com/apify/crawlee-python/commit/65095346a9d8b703b10c91e0510154c3c48a4176)) by [@Mantisus](https://github.com/Mantisus), closes [#1526](https://github.com/apify/crawlee-python/issues/1526)
 
 
 <!-- git-cliff-unreleased-end -->
+## [1.0.4](https://github.com/apify/crawlee-python/releases/tag/v1.0.4) (2025-10-24)
+
+### 🐛 Bug Fixes
+
+- Respect `enqueue_strategy` in `enqueue_links` ([#1505](https://github.com/apify/crawlee-python/pull/1505)) ([6ee04bc](https://github.com/apify/crawlee-python/commit/6ee04bc08c50a70f2e956a79d4ce5072a726c3a8)) by [@Mantisus](https://github.com/Mantisus), closes [#1504](https://github.com/apify/crawlee-python/issues/1504)
+- Exclude incorrect links before checking `robots.txt` ([#1502](https://github.com/apify/crawlee-python/pull/1502)) ([3273da5](https://github.com/apify/crawlee-python/commit/3273da5fee62ec9254666b376f382474c3532a56)) by [@Mantisus](https://github.com/Mantisus), closes [#1499](https://github.com/apify/crawlee-python/issues/1499)
+- Resolve compatibility issue between `SqlStorageClient` and `AdaptivePlaywrightCrawler` ([#1496](https://github.com/apify/crawlee-python/pull/1496)) ([ce172c4](https://github.com/apify/crawlee-python/commit/ce172c425a8643a1d4c919db4f5e5a6e47e91deb)) by [@Mantisus](https://github.com/Mantisus), closes [#1495](https://github.com/apify/crawlee-python/issues/1495)
+- Fix `BasicCrawler` statistics persistence ([#1490](https://github.com/apify/crawlee-python/pull/1490)) ([1eb1c19](https://github.com/apify/crawlee-python/commit/1eb1c19aa6f9dda4a0e3f7eda23f77a554f95076)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1501](https://github.com/apify/crawlee-python/issues/1501)
+- Save context state in result for `AdaptivePlaywrightCrawler` after isolated processing in `SubCrawler` ([#1488](https://github.com/apify/crawlee-python/pull/1488)) ([62b7c70](https://github.com/apify/crawlee-python/commit/62b7c70b54085fc65a660062028014f4502beba9)) by [@Mantisus](https://github.com/Mantisus), closes [#1483](https://github.com/apify/crawlee-python/issues/1483)
+
+
+## [1.0.3](https://github.com/apify/crawlee-python/releases/tag/v1.0.3) (2025-10-17)
+
+### 🐛 Bug Fixes
+
+- Add support for Pydantic v2.12 ([#1471](https://github.com/apify/crawlee-python/pull/1471)) ([35c1108](https://github.com/apify/crawlee-python/commit/35c110878c2f445a2866be2522ea8703e9b371dd)) by [@Mantisus](https://github.com/Mantisus), closes [#1464](https://github.com/apify/crawlee-python/issues/1464)
+- Fix database version warning message ([#1485](https://github.com/apify/crawlee-python/pull/1485)) ([18a545e](https://github.com/apify/crawlee-python/commit/18a545ee8add92e844acd0068f9cb8580a82e1c9)) by [@Mantisus](https://github.com/Mantisus)
+- Fix `reclaim_request` in `SqlRequestQueueClient` to correctly update the request state ([#1486](https://github.com/apify/crawlee-python/pull/1486)) ([1502469](https://github.com/apify/crawlee-python/commit/150246957f8f7f1ceb77bb77e3a02a903c50cae1)) by [@Mantisus](https://github.com/Mantisus), closes [#1484](https://github.com/apify/crawlee-python/issues/1484)
+- Fix `KeyValueStore.auto_saved_value` failing in some scenarios ([#1438](https://github.com/apify/crawlee-python/pull/1438)) ([b35dee7](https://github.com/apify/crawlee-python/commit/b35dee78180e57161b826641d45a61b8d8f6ef51)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1354](https://github.com/apify/crawlee-python/issues/1354)
+
+
+## [1.0.2](https://github.com/apify/crawlee-python/releases/tag/v1.0.2) (2025-10-08)
+
+### 🐛 Bug Fixes
+
+- Use Self type in the open() method of storage clients ([#1462](https://github.com/apify/crawlee-python/pull/1462)) ([4ec6f6c](https://github.com/apify/crawlee-python/commit/4ec6f6c08f81632197f602ff99151338b3eba6e7)) by [@janbuchar](https://github.com/janbuchar)
+- Add storages name validation ([#1457](https://github.com/apify/crawlee-python/pull/1457)) ([84de11a](https://github.com/apify/crawlee-python/commit/84de11a3a603503076f5b7df487c9abab68a9015)) by [@Mantisus](https://github.com/Mantisus), closes [#1434](https://github.com/apify/crawlee-python/issues/1434)
+- Pin pydantic version to &lt;2.12.0 to avoid compatibility issues ([#1467](https://github.com/apify/crawlee-python/pull/1467)) ([f11b86f](https://github.com/apify/crawlee-python/commit/f11b86f7ed57f98e83dc1b52f15f2017a919bf59)) by [@vdusek](https://github.com/vdusek)
+
+
+## [1.0.1](https://github.com/apify/crawlee-python/releases/tag/v1.0.1) (2025-10-06)
+
+### 🐛 Bug Fixes
+
+- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+- Update templates to handle optional httpx client ([#1440](https://github.com/apify/crawlee-python/pull/1440)) ([c087efd](https://github.com/apify/crawlee-python/commit/c087efd39baedf46ca3e5cae1ddc1acd6396e6c1)) by [@Pijukatel](https://github.com/Pijukatel)
+
+
 ## [1.0.0](https://github.com/apify/crawlee-python/releases/tag/v1.0.0) (2025-09-29)
 
+- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v1) for more details.
+- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v1) to ensure a smooth update.
+
 ### 🚀 Features
 
 - Add utility for load and parse Sitemap and `SitemapRequestLoader` ([#1169](https://github.com/apify/crawlee-python/pull/1169)) ([66599f8](https://github.com/apify/crawlee-python/commit/66599f8d085f3a8622e130019b6fdce2325737de)) by [@Mantisus](https://github.com/Mantisus), closes [#1161](https://github.com/apify/crawlee-python/issues/1161)
@@ -195,6 +240,9 @@ All notable changes to this project will be documented in this file.
 
 ## [0.6.0](https://github.com/apify/crawlee-python/releases/tag/v0.6.0) (2025-03-03)
 
+- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v06) for more details.
+- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v06) to ensure a smooth update.
+
 ### 🚀 Features
 
 - Integrate browserforge fingerprints ([#829](https://github.com/apify/crawlee-python/pull/829)) ([2b156b4](https://github.com/apify/crawlee-python/commit/2b156b4ba688f9111195422e6058dff30eb1f782)) by [@Pijukatel](https://github.com/Pijukatel), closes [#549](https://github.com/apify/crawlee-python/issues/549)
@@ -275,6 +323,9 @@ All notable changes to this project will be documented in this file.
 
 ## [0.5.0](https://github.com/apify/crawlee-python/releases/tag/v0.5.0) (2025-01-02)
 
+- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v05) for more details.
+- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v05) to ensure a smooth update.
+
 ### 🚀 Features
 
 - Add possibility to use None as no proxy in tiered proxies ([#760](https://github.com/apify/crawlee-python/pull/760)) ([0fbd017](https://github.com/apify/crawlee-python/commit/0fbd01723b9fe2e3410e0f358cab2f22848b08d0)) by [@Pijukatel](https://github.com/Pijukatel), closes [#687](https://github.com/apify/crawlee-python/issues/687)
@@ -366,6 +417,8 @@ All notable changes to this project will be documented in this file.
 
 ## [0.4.0](https://github.com/apify/crawlee-python/releases/tag/v0.4.0) (2024-11-01)
 
+- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v04) to ensure a smooth update.
+
 ### 🚀 Features
 
 - [**breaking**] Add headers in unique key computation ([#609](https://github.com/apify/crawlee-python/pull/609)) ([6c4746f](https://github.com/apify/crawlee-python/commit/6c4746fa8ff86952a812b32a1d70dc910e76b43e)) by [@Prathamesh010](https://github.com/Prathamesh010), closes [#548](https://github.com/apify/crawlee-python/issues/548)
@@ -475,6 +528,8 @@ All notable changes to this project will be documented in this file.
 
 ## [0.3.0](https://github.com/apify/crawlee-python/releases/tag/v0.3.0) (2024-08-27)
 
+- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v03) to ensure a smooth update.
+
 ### 🚀 Features
 
 - Implement ParselCrawler that adds support for Parsel ([#348](https://github.com/apify/crawlee-python/pull/348)) ([a3832e5](https://github.com/apify/crawlee-python/commit/a3832e527f022f32cce4a80055da3b7967b74522)) by [@asymness](https://github.com/asymness), closes [#335](https://github.com/apify/crawlee-python/issues/335)

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.1b7
+Version: 1.0.5b3
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

crawlee-1.0.5b3/docs/examples/code_examples/using_browser_profiles_chrome.py

@@ -0,0 +1,54 @@
+import asyncio
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+
+# Profile name to use (usually 'Default' for single profile setups)
+PROFILE_NAME = 'Default'
+
+# Paths to Chrome profiles in your system (example for Windows)
+# Use `chrome://version/` to find your profile path
+PROFILE_PATH = Path(Path.home(), 'AppData', 'Local', 'Google', 'Chrome', 'User Data')
+
+
+async def main() -> None:
+    # Create a temporary folder to copy the profile to
+    with TemporaryDirectory(prefix='crawlee-') as tmpdirname:
+        tmp_profile_dir = Path(tmpdirname)
+
+        # Copy the profile to a temporary folder
+        shutil.copytree(
+            PROFILE_PATH / PROFILE_NAME,
+            tmp_profile_dir / PROFILE_NAME,
+            dirs_exist_ok=True,
+        )
+
+        crawler = PlaywrightCrawler(
+            headless=False,
+            # Use the installed Chrome browser
+            browser_type='chrome',
+            # Disable fingerprints to preserve profile identity
+            fingerprint_generator=None,
+            # Set user data directory to temp folder
+            user_data_dir=tmp_profile_dir,
+            browser_launch_options={
+                # Slow down actions to mimic human behavior
+                'slow_mo': 200,
+                'args': [
+                    # Use the specified profile
+                    f'--profile-directory={PROFILE_NAME}',
+                ],
+            },
+        )
+
+        @crawler.router.default_handler
+        async def default_handler(context: PlaywrightCrawlingContext) -> None:
+            context.log.info(f'Visiting {context.request.url}')
+
+        await crawler.run(['https://crawlee.dev/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.5b3/docs/examples/code_examples/using_browser_profiles_firefox.py

@@ -0,0 +1,42 @@
+import asyncio
+from pathlib import Path
+
+from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+
+# Replace this with your actual Firefox profile name
+# Find it at about:profiles in Firefox
+PROFILE_NAME = 'your-profile-name-here'
+
+# Paths to Firefox profiles in your system (example for Windows)
+# Use `about:profiles` to find your profile path
+PROFILE_PATH = Path(
+    Path.home(), 'AppData', 'Roaming', 'Mozilla', 'Firefox', 'Profiles', PROFILE_NAME
+)
+
+
+async def main() -> None:
+    crawler = PlaywrightCrawler(
+        # Use Firefox browser type
+        browser_type='firefox',
+        # Disable fingerprints to use the profile as is
+        fingerprint_generator=None,
+        headless=False,
+        # Path to your Firefox profile
+        user_data_dir=PROFILE_PATH,
+        browser_launch_options={
+            'args': [
+                # Required to avoid version conflicts
+                '--allow-downgrade'
+            ]
+        },
+    )
+
+    @crawler.router.default_handler
+    async def default_handler(context: PlaywrightCrawlingContext) -> None:
+        context.log.info(f'Visiting {context.request.url}')
+
+    await crawler.run(['https://crawlee.dev/'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.5b3/docs/examples/using_browser_profile.mdx

@@ -0,0 +1,39 @@
+---
+id: using_browser_profile
+title: Using browser profile
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+import CodeBlock from '@theme/CodeBlock';
+
+import ChromeProfileExample from '!!raw-loader!./code_examples/using_browser_profiles_chrome.py';
+import FirefoxProfileExample from '!!raw-loader!./code_examples/using_browser_profiles_firefox.py';
+
+This example demonstrates how to run <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> using your local browser profile from [Chrome](https://www.google.com/intl/us/chrome/) or [Firefox](https://www.firefox.com/).
+
+Using browser profiles allows you to leverage existing login sessions, saved passwords, bookmarks, and other personalized browser data during crawling. This can be particularly useful for testing scenarios or when you need to access content that requires authentication.
+
+## Chrome browser
+
+To run <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> with your Chrome profile, you need to know the path to your profile files. You can find this information by entering `chrome://version/` as a URL in your Chrome browser. If you have multiple profiles, pay attention to the profile name - if you only have one profile, it's always `Default`.
+
+:::warning Profile access limitation
+Due to [Chrome's security policies](https://developer.chrome.com/blog/remote-debugging-port), automation cannot use your main browsing profile directly. The example copies your profile to a temporary location as a workaround.
+:::
+
+Make sure you don't have any running Chrome browser processes before running this code:
+
+<CodeBlock className="language-python" language="python">
+    {ChromeProfileExample}
+</CodeBlock>
+
+## Firefox browser
+
+To find the path to your Firefox profile, enter `about:profiles` as a URL in your Firefox browser. Unlike Chrome, you can use your standard profile path directly without copying it first.
+
+Make sure you don't have any running Firefox browser processes before running this code:
+
+<CodeBlock className="language-python" language="python">
+    {FirefoxProfileExample}
+</CodeBlock>

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/docs/guides/architecture_overview.mdx

@@ -291,7 +291,7 @@ Request loaders provide a subset of <ApiLink to="class/RequestQueue">`RequestQue
 
 - <ApiLink to="class/RequestLoader">`RequestLoader`</ApiLink> - Base interface for read-only access to a stream of requests, with capabilities like fetching the next request, marking as handled, and status checking.
 - <ApiLink to="class/RequestList">`RequestList`</ApiLink> - Lightweight in-memory implementation of `RequestLoader` for managing static lists of URLs.
-- <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> - Specialized loader for reading URLs from XML sitemaps with filtering capabilities.
+- <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> - A specialized loader that reads URLs from XML and plain-text sitemaps following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html) with filtering capabilities.
 
 ### Request managers
 

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/docs/guides/avoid_blocking.mdx

@@ -25,7 +25,7 @@ Changing browser fingerprints can be a tedious job. Luckily, Crawlee provides th
     {PlaywrightDefaultFingerprintGenerator}
 </RunnableCodeBlock>
 
-In certain cases we want to narrow down the fingerprints used - e.g. specify a certain operating system, locale or browser. This is also possible with Crawlee - the crawler can have the generation algorithm customized to reflect the particular browser version and many more. For description of fingerprint generation options please see <ApiLink to="class/HeaderGeneratorOptions">`HeaderGeneratorOptions`</ApiLink>, <ApiLink to="class/ScreenOptions">`ScreenOptions`</ApiLink> and <ApiLink to="class/BrowserforgeFingerprintGenerator#__init__">`DefaultFingerprintGenerator.__init__`</ApiLink>  See the example bellow:
+In certain cases we want to narrow down the fingerprints used - e.g. specify a certain operating system, locale or browser. This is also possible with Crawlee - the crawler can have the generation algorithm customized to reflect the particular browser version and many more. For description of fingerprint generation options please see <ApiLink to="class/HeaderGeneratorOptions">`HeaderGeneratorOptions`</ApiLink>, <ApiLink to="class/ScreenOptions">`ScreenOptions`</ApiLink> and <ApiLink to="class/BrowserforgeFingerprintGenerator#__init__">`DefaultFingerprintGenerator.__init__`</ApiLink>  See the example below:
 
 <CodeBlock className="language-python">
     {PlaywrightDefaultFingerprintGeneratorWithArgs}

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/docs/guides/request_loaders.mdx

@@ -31,7 +31,7 @@ The [`request_loaders`](https://github.com/apify/crawlee-python/tree/master/src/
 And specific request loader implementations:
 
 - <ApiLink to="class/RequestList">`RequestList`</ApiLink>: A lightweight implementation for managing a static list of URLs.
-- <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink>: A specialized loader that reads URLs from XML sitemaps with filtering capabilities.
+- <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink>: A specialized loader that reads URLs from XML and plain-text sitemaps following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html) with filtering capabilities.
 
 Below is a class diagram that illustrates the relationships between these components and the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>:
 
@@ -130,7 +130,13 @@ To enable persistence, provide `persist_state_key` and optionally `persist_reque
 
 ### Sitemap request loader
 
-The <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> is a specialized request loader that reads URLs from XML sitemaps. It's particularly useful when you want to crawl a website systematically by following its sitemap structure. The loader supports filtering URLs using glob patterns and regular expressions, allowing you to include or exclude specific types of URLs. The <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> provides streaming processing of sitemaps, ensuring efficient memory usage without loading the entire sitemap into memory.
+The <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> is a specialized request loader that reads URLs from sitemaps following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats. It's particularly useful when you want to crawl a website systematically by following its sitemap structure.
+
+:::note
+The `SitemapRequestLoader` is designed specifically for sitemaps that follow the standard Sitemaps protocol. HTML pages containing links are not supported by this loader - those should be handled by regular crawlers using the `enqueue_links` functionality.
+:::
+
+The loader supports filtering URLs using glob patterns and regular expressions, allowing you to include or exclude specific types of URLs. The <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> provides streaming processing of sitemaps, ensuring efficient memory usage without loading the entire sitemap into memory.
 
 <RunnableCodeBlock className="language-python" language="python">
     {SitemapExample}

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/docs/introduction/09_running_in_cloud.mdx

@@ -50,7 +50,7 @@ apify login
 
 Now that you have your account set up, you will need to adjust the code a tiny bit. We will use the [Apify SDK](https://docs.apify.com/sdk/python/), which will help us to wire the Crawlee storages (like the [`RequestQueue`](https://docs.apify.com/sdk/python/reference/class/RequestQueue)) to their Apify platform counterparts - otherwise Crawlee would keep things only in memory.
 
-Open your `src/main.py` file, and wrap everyting in your `main` function with the [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) context manager. Your code should look like this:
+Open your `src/main.py` file, and wrap everything in your `main` function with the [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) context manager. Your code should look like this:
 
 <CodeBlock className="language-python" title="src/main.py">
     {MainExample}

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/docs/upgrading/upgrading_to_v1.md

@@ -333,3 +333,7 @@ async def main() -> None:
 
     await crawler.run(['https://crawlee.dev/'])
 ```
+
+### New storage naming restrictions
+
+We've introduced naming restrictions for storages to ensure compatibility with Apify Platform requirements and prevent potential conflicts. Storage names may include only letters (a–z, A–Z), digits (0–9), and hyphens (-), with hyphens allowed only in the middle of the name (for example, my-storage-1).

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.1b7"
+version = "1.0.5b3"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -107,14 +107,14 @@ dev = [
     "pytest-timeout~=2.4.0",
     "pytest-xdist~=3.8.0",
     "pytest~=8.4.0",
-    "ruff~=0.13.0",
+    "ruff~=0.14.0",
     "setuptools", # setuptools are used by pytest, but not explicitly required
     "types-beautifulsoup4~=4.12.0.20240229",
     "types-cachetools~=6.2.0.20250827",
     "types-colorama~=0.4.15.20240106",
     "types-psutil~=7.0.0.20250218",
     "types-python-dateutil~=2.9.0.20240316",
-    "uvicorn[standard]~=0.37.0",
+    "uvicorn[standard]~=0.35.0", # https://github.com/apify/crawlee-python/issues/1441
 ]
 
 [tool.hatch.build.targets.wheel]

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/src/crawlee/_request.py

@@ -185,9 +185,6 @@ class Request(BaseModel):
     method: HttpMethod = 'GET'
     """HTTP request method."""
 
-    headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)] = HttpHeaders()
-    """HTTP request headers."""
-
     payload: Annotated[
         HttpPayload | None,
         BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v),
@@ -195,23 +192,37 @@ class Request(BaseModel):
     ] = None
     """HTTP request payload."""
 
-    user_data: Annotated[
-        dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
-        Field(alias='userData', default_factory=lambda: UserData()),
-        PlainValidator(user_data_adapter.validate_python),
-        PlainSerializer(
-            lambda instance: user_data_adapter.dump_python(
-                instance,
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-                exclude_defaults=True,
-            )
-        ),
-    ] = {}
-    """Custom user data assigned to the request. Use this to save any request related data to the
-    request's scope, keeping them accessible on retries, failures etc.
-    """
+    # Workaround for pydantic 2.12 and mypy type checking issue for Annotated with default_factory
+    if TYPE_CHECKING:
+        headers: HttpHeaders = HttpHeaders()
+        """HTTP request headers."""
+
+        user_data: dict[str, JsonSerializable] = {}
+        """Custom user data assigned to the request. Use this to save any request related data to the
+        request's scope, keeping them accessible on retries, failures etc.
+        """
+
+    else:
+        headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)]
+        """HTTP request headers."""
+
+        user_data: Annotated[
+            dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
+            Field(alias='userData', default_factory=lambda: UserData()),
+            PlainValidator(user_data_adapter.validate_python),
+            PlainSerializer(
+                lambda instance: user_data_adapter.dump_python(
+                    instance,
+                    by_alias=True,
+                    exclude_none=True,
+                    exclude_unset=True,
+                    exclude_defaults=True,
+                )
+            ),
+        ]
+        """Custom user data assigned to the request. Use this to save any request related data to the
+        request's scope, keeping them accessible on retries, failures etc.
+        """
 
     retry_count: Annotated[int, Field(alias='retryCount')] = 0
     """Number of times the request has been retried."""

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/src/crawlee/_service_locator.py

@@ -38,7 +38,7 @@ class ServiceLocator:
     def get_configuration(self) -> Configuration:
         """Get the configuration."""
         if self._configuration is None:
-            logger.warning('No configuration set, implicitly creating and using default Configuration.')
+            logger.debug('No configuration set, implicitly creating and using default Configuration.')
             self._configuration = Configuration()
 
         return self._configuration
@@ -63,9 +63,9 @@ class ServiceLocator:
     def get_event_manager(self) -> EventManager:
         """Get the event manager."""
         if self._event_manager is None:
-            logger.warning('No event manager set, implicitly creating and using default LocalEventManager.')
+            logger.debug('No event manager set, implicitly creating and using default LocalEventManager.')
             if self._configuration is None:
-                logger.warning(
+                logger.debug(
                     'Implicit creation of event manager will implicitly set configuration as side effect. '
                     'It is advised to explicitly first set the configuration instead.'
                 )
@@ -93,7 +93,7 @@ class ServiceLocator:
     def get_storage_client(self) -> StorageClient:
         """Get the storage client."""
         if self._storage_client is None:
-            logger.warning('No storage client set, implicitly creating and using default FileSystemStorageClient.')
+            logger.debug('No storage client set, implicitly creating and using default FileSystemStorageClient.')
             if self._configuration is None:
                 logger.warning(
                     'Implicit creation of storage client will implicitly set configuration as side effect. '

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/src/crawlee/_types.py

@@ -3,17 +3,7 @@ from __future__ import annotations
 import dataclasses
 from collections.abc import Callable, Iterator, Mapping
 from dataclasses import dataclass
-from typing import (
-    TYPE_CHECKING,
-    Annotated,
-    Any,
-    Literal,
-    Protocol,
-    TypedDict,
-    TypeVar,
-    cast,
-    overload,
-)
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol, TypedDict, TypeVar, cast, overload
 
 from pydantic import ConfigDict, Field, PlainValidator, RootModel
 
@@ -71,11 +61,15 @@ class HttpHeaders(RootModel, Mapping[str, str]):
 
     model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
-    root: Annotated[
-        dict[str, str],
-        PlainValidator(lambda value: _normalize_headers(value)),
-        Field(default_factory=dict),
-    ] = {}
+    # Workaround for pydantic 2.12 and mypy type checking issue for Annotated with default_factory
+    if TYPE_CHECKING:
+        root: dict[str, str] = {}
+    else:
+        root: Annotated[
+            dict[str, str],
+            PlainValidator(lambda value: _normalize_headers(value)),
+            Field(default_factory=dict),
+        ]
 
     def __getitem__(self, key: str) -> str:
         return self.root[key.lower()]

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/src/crawlee/_utils/recoverable_state.py

@@ -4,12 +4,14 @@ from typing import TYPE_CHECKING, Generic, Literal, TypeVar
 
 from pydantic import BaseModel
 
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.events._types import Event, EventPersistStateData
 
 if TYPE_CHECKING:
     import logging
+    from collections.abc import Callable, Coroutine
 
-    from crawlee.storages._key_value_store import KeyValueStore
+    from crawlee.storages import KeyValueStore
 
 TStateModel = TypeVar('TStateModel', bound=BaseModel)
 
@@ -37,6 +39,7 @@ class RecoverableState(Generic[TStateModel]):
         persistence_enabled: Literal[True, False, 'explicit_only'] = False,
         persist_state_kvs_name: str | None = None,
         persist_state_kvs_id: str | None = None,
+        persist_state_kvs_factory: Callable[[], Coroutine[None, None, KeyValueStore]] | None = None,
         logger: logging.Logger,
     ) -> None:
         """Initialize a new recoverable state object.
@@ -51,16 +54,40 @@ class RecoverableState(Generic[TStateModel]):
                 If neither a name nor and id are supplied, the default store will be used.
             persist_state_kvs_id: The identifier of the KeyValueStore to use for persistence.
                 If neither a name nor and id are supplied, the default store will be used.
+            persist_state_kvs_factory: Factory that can be awaited to create KeyValueStore to use for persistence. If
+                not provided, a system-wide KeyValueStore will be used, based on service locator configuration.
             logger: A logger instance for logging operations related to state persistence
         """
+        raise_if_too_many_kwargs(
+            persist_state_kvs_name=persist_state_kvs_name,
+            persist_state_kvs_id=persist_state_kvs_id,
+            persist_state_kvs_factory=persist_state_kvs_factory,
+        )
+        if not persist_state_kvs_factory:
+            logger.debug(
+                'No explicit key_value_store set for recoverable state. Recovery will use a system-wide KeyValueStore '
+                'based on service_locator configuration, potentially calling service_locator.set_storage_client in the '
+                'process. It is recommended to initialize RecoverableState with explicit key_value_store to avoid '
+                'global side effects.'
+            )
+
         self._default_state = default_state
         self._state_type: type[TStateModel] = self._default_state.__class__
         self._state: TStateModel | None = None
         self._persistence_enabled = persistence_enabled
         self._persist_state_key = persist_state_key
-        self._persist_state_kvs_name = persist_state_kvs_name
-        self._persist_state_kvs_id = persist_state_kvs_id
-        self._key_value_store: 'KeyValueStore | None' = None  # noqa: UP037
+        if persist_state_kvs_factory is None:
+
+            async def kvs_factory() -> KeyValueStore:
+                from crawlee.storages import KeyValueStore  # noqa: PLC0415 avoid circular import
+
+                return await KeyValueStore.open(name=persist_state_kvs_name, id=persist_state_kvs_id)
+
+            self._persist_state_kvs_factory = kvs_factory
+        else:
+            self._persist_state_kvs_factory = persist_state_kvs_factory
+
+        self._key_value_store: KeyValueStore | None = None
         self._log = logger
 
     async def initialize(self) -> TStateModel:
@@ -77,11 +104,8 @@ class RecoverableState(Generic[TStateModel]):
             return self.current_value
 
         # Import here to avoid circular imports.
-        from crawlee.storages._key_value_store import KeyValueStore  # noqa: PLC0415
 
-        self._key_value_store = await KeyValueStore.open(
-            name=self._persist_state_kvs_name, id=self._persist_state_kvs_id
-        )
+        self._key_value_store = await self._persist_state_kvs_factory()
 
         await self._load_saved_state()
 

{crawlee-1.0.1b7 → crawlee-1.0.5b3}/src/crawlee/_utils/recurring_task.py

@@ -7,6 +7,9 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Callable
     from datetime import timedelta
+    from types import TracebackType
+
+    from typing_extensions import Self
 
 logger = getLogger(__name__)
 
@@ -26,6 +29,18 @@ class RecurringTask:
         self.delay = delay
         self.task: asyncio.Task | None = None
 
+    async def __aenter__(self) -> Self:
+        self.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        await self.stop()
+
     async def _wrapper(self) -> None:
         """Continuously execute the provided function with the specified delay.