crawlee 1.0.0rc1__tar.gz → 1.0.1__tar.gz

{crawlee-1.0.0rc1 → crawlee-1.0.1}/.github/workflows/build_and_deploy_docs.yaml

@@ -30,12 +30,12 @@ jobs:
           ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
 
       - name: Set up Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: ${{ env.NODE_VERSION }}
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
@@ -59,7 +59,7 @@ jobs:
         uses: actions/configure-pages@v5
 
       - name: Upload GitHub Pages artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v4
         with:
           path: ./website/build
 

{crawlee-1.0.0rc1 → crawlee-1.0.1}/.github/workflows/check_pr_title.yaml

@@ -9,6 +9,6 @@ jobs:
     name: Check PR title
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v6.1.0
+      - uses: amannn/action-semantic-pull-request@v6.1.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{crawlee-1.0.0rc1 → crawlee-1.0.1}/.github/workflows/pre_release.yaml

@@ -37,7 +37,7 @@ jobs:
     name: Wait for code checks to pass
     runs-on: ubuntu-latest
     steps:
-      - uses: lewagon/wait-on-check-action@v1.4.0
+      - uses: lewagon/wait-on-check-action@v1.4.1
         with:
           ref: ${{ github.ref }}
           repo-token: ${{ secrets.GITHUB_TOKEN }}

{crawlee-1.0.0rc1 → crawlee-1.0.1}/.github/workflows/templates_e2e_tests.yaml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v5
 
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: ${{ env.NODE_VERSION }}
 
@@ -35,7 +35,7 @@ jobs:
         run: npm install -g apify-cli
 
       - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 

{crawlee-1.0.0rc1 → crawlee-1.0.1}/.github/workflows/update_new_issue.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       # Add the "t-tooling" label to all new issues
-      - uses: actions/github-script@v7
+      - uses: actions/github-script@v8
         with:
           script: |
             github.rest.issues.addLabels({

{crawlee-1.0.0rc1 → crawlee-1.0.1}/CHANGELOG.md

@@ -2,7 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
-## [1.0.0rc1](https://github.com/apify/crawlee-python/releases/tag/v1.0.0rc1) (2025-08-22)
+## [1.0.1](https://github.com/apify/crawlee-python/releases/tag/v1.0.1) (2025-10-06)
+
+### 🐛 Bug Fixes
+
+- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+- Update templates to handle optional httpx client ([#1440](https://github.com/apify/crawlee-python/pull/1440)) ([c087efd](https://github.com/apify/crawlee-python/commit/c087efd39baedf46ca3e5cae1ddc1acd6396e6c1)) by [@Pijukatel](https://github.com/Pijukatel)
+
+
+## [1.0.0](https://github.com/apify/crawlee-python/releases/tag/v1.0.0) (2025-09-29)
 
 ### 🚀 Features
 
@@ -17,6 +25,10 @@ All notable changes to this project will be documented in this file.
 - Add `impit` option for Crawlee CLI ([#1312](https://github.com/apify/crawlee-python/pull/1312)) ([508d7ce](https://github.com/apify/crawlee-python/commit/508d7ce4d998f37ab2adcf9c057c3c635a69f863)) by [@Mantisus](https://github.com/Mantisus)
 - Persist RequestList state ([#1274](https://github.com/apify/crawlee-python/pull/1274)) ([cc68014](https://github.com/apify/crawlee-python/commit/cc680147ba3cc8b35b9da70274e53e6f5dd92434)) by [@janbuchar](https://github.com/janbuchar), closes [#99](https://github.com/apify/crawlee-python/issues/99)
 - Persist `DefaultRenderingTypePredictor` state ([#1340](https://github.com/apify/crawlee-python/pull/1340)) ([fad4c25](https://github.com/apify/crawlee-python/commit/fad4c25fc712915c4a45b24e3290b6f5dbd8a683)) by [@Mantisus](https://github.com/Mantisus), closes [#1272](https://github.com/apify/crawlee-python/issues/1272)
+- Persist the `SitemapRequestLoader` state ([#1347](https://github.com/apify/crawlee-python/pull/1347)) ([27ef9ad](https://github.com/apify/crawlee-python/commit/27ef9ad194552ea9f1321d91a7a52054be9a8a51)) by [@Mantisus](https://github.com/Mantisus), closes [#1269](https://github.com/apify/crawlee-python/issues/1269)
+- Add support for NDU storages ([#1401](https://github.com/apify/crawlee-python/pull/1401)) ([5dbd212](https://github.com/apify/crawlee-python/commit/5dbd212663e7abc37535713f4c6e3a5bbf30a12e)) by [@vdusek](https://github.com/vdusek), closes [#1175](https://github.com/apify/crawlee-python/issues/1175)
+- Add RQ id, name, alias args to `add_requests` and `enqueue_links` methods ([#1413](https://github.com/apify/crawlee-python/pull/1413)) ([1cae2bc](https://github.com/apify/crawlee-python/commit/1cae2bca0b1508fcb3cb419dc239caf33e20a7ef)) by [@Mantisus](https://github.com/Mantisus), closes [#1402](https://github.com/apify/crawlee-python/issues/1402)
+- Add `SqlStorageClient` based on `sqlalchemy` v2+ ([#1339](https://github.com/apify/crawlee-python/pull/1339)) ([07c75a0](https://github.com/apify/crawlee-python/commit/07c75a078b443b58bfaaeb72eb2aa1439458dc47)) by [@Mantisus](https://github.com/Mantisus), closes [#307](https://github.com/apify/crawlee-python/issues/307)
 
 ### 🐛 Bug Fixes
 
@@ -27,6 +39,9 @@ All notable changes to this project will be documented in this file.
 - Fix `timeout` for `stream` method in `ImpitHttpClient` ([#1352](https://github.com/apify/crawlee-python/pull/1352)) ([54b693b](https://github.com/apify/crawlee-python/commit/54b693b838f135a596e1e9493b565bc558b19a3a)) by [@Mantisus](https://github.com/Mantisus)
 - Include reason in the session rotation warning logs ([#1363](https://github.com/apify/crawlee-python/pull/1363)) ([d6d7a45](https://github.com/apify/crawlee-python/commit/d6d7a45dd64a906419d9552c45062d726cbb1a0f)) by [@vdusek](https://github.com/vdusek), closes [#1318](https://github.com/apify/crawlee-python/issues/1318)
 - Improve crawler statistics logging ([#1364](https://github.com/apify/crawlee-python/pull/1364)) ([1eb6da5](https://github.com/apify/crawlee-python/commit/1eb6da5dd85870124593dcad877284ccaed9c0ce)) by [@vdusek](https://github.com/vdusek), closes [#1317](https://github.com/apify/crawlee-python/issues/1317)
+- Do not add a request that is already in progress to `MemoryRequestQueueClient` ([#1384](https://github.com/apify/crawlee-python/pull/1384)) ([3af326c](https://github.com/apify/crawlee-python/commit/3af326c9dfa8fffd56a42ca42981374613739e39)) by [@Mantisus](https://github.com/Mantisus), closes [#1383](https://github.com/apify/crawlee-python/issues/1383)
+- Save `RequestQueueState` for `FileSystemRequestQueueClient` in default KVS ([#1411](https://github.com/apify/crawlee-python/pull/1411)) ([6ee60a0](https://github.com/apify/crawlee-python/commit/6ee60a08ac1f9414e1b792f4935cc3799cb5089a)) by [@Mantisus](https://github.com/Mantisus), closes [#1410](https://github.com/apify/crawlee-python/issues/1410)
+- Set default desired concurrency for non-browser crawlers to 10 ([#1419](https://github.com/apify/crawlee-python/pull/1419)) ([1cc9401](https://github.com/apify/crawlee-python/commit/1cc940197600d2539bda967880d7f9d241eb8c3e)) by [@vdusek](https://github.com/vdusek)
 
 ### Refactor
 
@@ -36,6 +51,8 @@ All notable changes to this project will be documented in this file.
 - [**breaking**] Replace `HttpxHttpClient` with `ImpitHttpClient` as default HTTP client ([#1307](https://github.com/apify/crawlee-python/pull/1307)) ([c803a97](https://github.com/apify/crawlee-python/commit/c803a976776a76846866d533e3a3ee8144e248c4)) by [@Mantisus](https://github.com/Mantisus), closes [#1079](https://github.com/apify/crawlee-python/issues/1079)
 - [**breaking**] Change Dataset unwind parameter to accept list of strings ([#1357](https://github.com/apify/crawlee-python/pull/1357)) ([862a203](https://github.com/apify/crawlee-python/commit/862a20398f00fe91802fe7a1ccd58b05aee053a1)) by [@vdusek](https://github.com/vdusek)
 - [**breaking**] Remove `Request.id` field ([#1366](https://github.com/apify/crawlee-python/pull/1366)) ([32f3580](https://github.com/apify/crawlee-python/commit/32f3580e9775a871924ab1233085d0c549c4cd52)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1358](https://github.com/apify/crawlee-python/issues/1358)
+- [**breaking**] Refactor storage creation and caching, configuration and services ([#1386](https://github.com/apify/crawlee-python/pull/1386)) ([04649bd](https://github.com/apify/crawlee-python/commit/04649bde60d46b2bc18ae4f6e3fd9667d02a9cef)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1379](https://github.com/apify/crawlee-python/issues/1379)
+
 
 
 ## [0.6.12](https://github.com/apify/crawlee-python/releases/tag/v0.6.12) (2025-07-30)

{crawlee-1.0.0rc1 → crawlee-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.0rc1
+Version: 1.0.1
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -227,12 +227,12 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
-Requires-Dist: impit>=0.5.2
+Requires-Dist: impit>=0.6.1
 Requires-Dist: more-itertools>=10.2.0
 Requires-Dist: protego>=0.5.0
 Requires-Dist: psutil>=6.0.0
-Requires-Dist: pydantic!=2.10.0,!=2.10.1,!=2.10.2,>=2.8.0
 Requires-Dist: pydantic-settings!=2.7.0,!=2.7.1,!=2.8.0,>=2.2.0
+Requires-Dist: pydantic>=2.11.0
 Requires-Dist: pyee>=9.0.0
 Requires-Dist: tldextract>=5.1.0
 Requires-Dist: typing-extensions>=4.1.0
@@ -244,7 +244,9 @@ Requires-Dist: jaro-winkler>=2.0.3; extra == 'adaptive-crawler'
 Requires-Dist: playwright>=1.27.0; extra == 'adaptive-crawler'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
+Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -263,6 +265,7 @@ Requires-Dist: parsel>=1.10.0; extra == 'all'
 Requires-Dist: playwright>=1.27.0; extra == 'all'
 Requires-Dist: rich>=13.9.0; extra == 'all'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'all'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: typer>=0.12.0; extra == 'all'
 Requires-Dist: wrapt>=1.17.0; extra == 'all'
 Provides-Extra: beautifulsoup
@@ -293,6 +296,12 @@ Provides-Extra: playwright
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'playwright'
 Requires-Dist: browserforge>=1.2.3; extra == 'playwright'
 Requires-Dist: playwright>=1.27.0; extra == 'playwright'
+Provides-Extra: sql-postgres
+Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
+Provides-Extra: sql-sqlite
+Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-sqlite'
 Description-Content-Type: text/markdown
 
 <h1 align="center">
@@ -327,8 +336,6 @@ Description-Content-Type: text/markdown
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
 
-> 🚀 Crawlee for Python is open to early adopters!
-
 Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈

{crawlee-1.0.0rc1 → crawlee-1.0.1}/README.md

@@ -30,8 +30,6 @@
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
 
-> 🚀 Crawlee for Python is open to early adopters!
-
 Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈

{crawlee-1.0.0rc1 → crawlee-1.0.1}/docs/guides/code_examples/playwright_crawler_adaptive/handler.py

@@ -5,7 +5,7 @@ from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawli
 
 
 async def main() -> None:
-    crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser()
+    crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
 
     @crawler.router.default_handler
     async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:

{crawlee-1.0.0rc1 → crawlee-1.0.1}/docs/guides/code_examples/request_loaders/rl_basic_example.py

@@ -18,6 +18,7 @@ async def main() -> None:
     # Fetch and process requests from the queue.
     while request := await request_list.fetch_next_request():
         # Do something with it...
+        print(f'Processing {request.url}')
 
         # And mark it as handled.
         await request_list.mark_request_as_handled(request)

crawlee-1.0.1/docs/guides/code_examples/request_loaders/rl_basic_example_with_persist.py

@@ -0,0 +1,46 @@
+import asyncio
+import logging
+
+from crawlee import service_locator
+from crawlee.request_loaders import RequestList
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
+logger = logging.getLogger(__name__)
+
+
+# Disable clearing the `KeyValueStore` on each run.
+# This is necessary so that the state keys are not cleared at startup.
+# The recommended way to achieve this behavior is setting the environment variable
+# `CRAWLEE_PURGE_ON_START=0`
+configuration = service_locator.get_configuration()
+configuration.purge_on_start = False
+
+
+async def main() -> None:
+    # Open the request list, if it does not exist, it will be created.
+    # Leave name empty to use the default request list.
+    request_list = RequestList(
+        name='my-request-list',
+        requests=[
+            'https://apify.com/',
+            'https://crawlee.dev/',
+            'https://crawlee.dev/python/',
+        ],
+        # Enable persistence
+        persist_state_key='my-persist-state',
+        persist_requests_key='my-persist-requests',
+    )
+
+    # We receive only one request.
+    # Each time you run it, it will be a new request until you exhaust the `RequestList`.
+    request = await request_list.fetch_next_request()
+    if request:
+        logger.info(f'Processing request: {request.url}')
+        # Do something with it...
+
+        # And mark it as handled.
+        await request_list.mark_request_as_handled(request)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

{crawlee-1.0.0rc1 → crawlee-1.0.1}/docs/guides/code_examples/request_loaders/rl_tandem_example.py

@@ -8,9 +8,11 @@ async def main() -> None:
     # Create a static request list.
     request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
 
+    # highlight-start
     # Convert the request list to a request manager using the to_tandem method.
     # It is a tandem with the default request queue.
     request_manager = await request_list.to_tandem()
+    # highlight-end
 
     # Create a crawler and pass the request manager to it.
     crawler = ParselCrawler(
@@ -20,9 +22,20 @@ async def main() -> None:
 
     @crawler.router.default_handler
     async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
         # New links will be enqueued directly to the queue.
         await context.enqueue_links()
 
+        # Extract data using Parsel's XPath and CSS selectors.
+        data = {
+            'url': context.request.url,
+            'title': context.selector.xpath('//title/text()').get(),
+        }
+
+        # Push extracted data to the dataset.
+        await context.push_data(data)
+
     await crawler.run()
 
 

{crawlee-1.0.0rc1 → crawlee-1.0.1}/docs/guides/code_examples/request_loaders/rl_tandem_example_explicit.py

@@ -23,9 +23,20 @@ async def main() -> None:
 
     @crawler.router.default_handler
     async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
         # New links will be enqueued directly to the queue.
         await context.enqueue_links()
 
+        # Extract data using Parsel's XPath and CSS selectors.
+        data = {
+            'url': context.request.url,
+            'title': context.selector.xpath('//title/text()').get(),
+        }
+
+        # Push extracted data to the dataset.
+        await context.push_data(data)
+
     await crawler.run()
 
 

crawlee-1.0.1/docs/guides/code_examples/request_loaders/sitemap_basic_example.py

@@ -0,0 +1,30 @@
+import asyncio
+import re
+
+from crawlee.http_clients import ImpitHttpClient
+from crawlee.request_loaders import SitemapRequestLoader
+
+
+async def main() -> None:
+    # Create an HTTP client for fetching the sitemap.
+    http_client = ImpitHttpClient()
+
+    # Create a sitemap request loader with filtering rules.
+    sitemap_loader = SitemapRequestLoader(
+        sitemap_urls=['https://crawlee.dev/sitemap.xml'],
+        http_client=http_client,
+        include=[re.compile(r'.*docs.*')],  # Only include URLs containing 'docs'.
+        max_buffer_size=500,  # Keep up to 500 URLs in memory before processing.
+    )
+
+    # We work with the loader until we process all relevant links from the sitemap.
+    while request := await sitemap_loader.fetch_next_request():
+        # Do something with it...
+        print(f'Processing {request.url}')
+
+        # And mark it as handled.
+        await sitemap_loader.mark_request_as_handled(request)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.1/docs/guides/code_examples/request_loaders/sitemap_example_with_persist.py

@@ -0,0 +1,45 @@
+import asyncio
+import logging
+
+from crawlee import service_locator
+from crawlee.http_clients import ImpitHttpClient
+from crawlee.request_loaders import SitemapRequestLoader
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
+logger = logging.getLogger(__name__)
+
+
+# Disable clearing the `KeyValueStore` on each run.
+# This is necessary so that the state keys are not cleared at startup.
+# The recommended way to achieve this behavior is setting the environment variable
+# `CRAWLEE_PURGE_ON_START=0`
+configuration = service_locator.get_configuration()
+configuration.purge_on_start = False
+
+
+async def main() -> None:
+    # Create an HTTP client for fetching sitemaps
+    # Use the context manager for `SitemapRequestLoader` to correctly save the state when
+    # the work is completed.
+    async with (
+        ImpitHttpClient() as http_client,
+        SitemapRequestLoader(
+            sitemap_urls=['https://crawlee.dev/sitemap.xml'],
+            http_client=http_client,
+            # Enable persistence
+            persist_state_key='my-persist-state',
+        ) as sitemap_loader,
+    ):
+        # We receive only one request.
+        # Each time you run it, it will be a new request until you exhaust the sitemap.
+        request = await sitemap_loader.fetch_next_request()
+        if request:
+            logger.info(f'Processing request: {request.url}')
+            # Do something with it...
+
+            # And mark it as handled.
+            await sitemap_loader.mark_request_as_handled(request)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.1/docs/guides/code_examples/request_loaders/sitemap_tandem_example.py

@@ -0,0 +1,53 @@
+import asyncio
+import re
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.http_clients import ImpitHttpClient
+from crawlee.request_loaders import SitemapRequestLoader
+
+
+async def main() -> None:
+    # Create an HTTP client for fetching the sitemap.
+    http_client = ImpitHttpClient()
+
+    # Create a sitemap request loader with filtering rules.
+    sitemap_loader = SitemapRequestLoader(
+        sitemap_urls=['https://crawlee.dev/sitemap.xml'],
+        http_client=http_client,
+        include=[re.compile(r'.*docs.*')],  # Only include URLs containing 'docs'.
+        max_buffer_size=500,  # Keep up to 500 URLs in memory before processing.
+    )
+
+    # highlight-start
+    # Convert the sitemap loader into a request manager linked
+    # to the default request queue.
+    request_manager = await sitemap_loader.to_tandem()
+    # highlight-end
+
+    # Create a crawler and pass the request manager to it.
+    crawler = ParselCrawler(
+        request_manager=request_manager,
+        max_requests_per_crawl=10,  # Limit the max requests per crawl.
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
+        # New links will be enqueued directly to the queue.
+        await context.enqueue_links()
+
+        # Extract data using Parsel's XPath and CSS selectors.
+        data = {
+            'url': context.request.url,
+            'title': context.selector.xpath('//title/text()').get(),
+        }
+
+        # Push extracted data to the dataset.
+        await context.push_data(data)
+
+    await crawler.run()
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.1/docs/guides/code_examples/request_loaders/sitemap_tandem_example_explicit.py

@@ -0,0 +1,54 @@
+import asyncio
+import re
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.http_clients import ImpitHttpClient
+from crawlee.request_loaders import RequestManagerTandem, SitemapRequestLoader
+from crawlee.storages import RequestQueue
+
+
+async def main() -> None:
+    # Create an HTTP client for fetching the sitemap.
+    http_client = ImpitHttpClient()
+
+    # Create a sitemap request loader with filtering rules.
+    sitemap_loader = SitemapRequestLoader(
+        sitemap_urls=['https://crawlee.dev/sitemap.xml'],
+        http_client=http_client,
+        include=[re.compile(r'.*docs.*')],  # Only include URLs containing 'docs'.
+        max_buffer_size=500,  # Keep up to 500 URLs in memory before processing.
+    )
+
+    # Open the default request queue.
+    request_queue = await RequestQueue.open()
+
+    # And combine them together to a single request manager.
+    request_manager = RequestManagerTandem(sitemap_loader, request_queue)
+
+    # Create a crawler and pass the request manager to it.
+    crawler = ParselCrawler(
+        request_manager=request_manager,
+        max_requests_per_crawl=10,  # Limit the max requests per crawl.
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        context.log.info(f'Processing {context.request.url}')
+
+        # New links will be enqueued directly to the queue.
+        await context.enqueue_links()
+
+        # Extract data using Parsel's XPath and CSS selectors.
+        data = {
+            'url': context.request.url,
+            'title': context.selector.xpath('//title/text()').get(),
+        }
+
+        # Push extracted data to the dataset.
+        await context.push_data(data)
+
+    await crawler.run()
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.1/docs/guides/code_examples/service_locator/service_storage_configuration.py

@@ -0,0 +1,30 @@
+import asyncio
+from datetime import timedelta
+
+from crawlee import service_locator
+from crawlee.configuration import Configuration
+from crawlee.storage_clients import MemoryStorageClient
+from crawlee.storages import Dataset
+
+
+async def main() -> None:
+    configuration = Configuration(
+        log_level='DEBUG',
+        headless=False,
+        persist_state_interval=timedelta(seconds=30),
+    )
+    # Set the custom configuration as the global default configuration.
+    service_locator.set_configuration(configuration)
+
+    # Use the global defaults when creating the dataset (or other storage).
+    dataset_1 = await Dataset.open()
+
+    # Or set explicitly specific configuration if
+    # you do not want to rely on global defaults.
+    dataset_2 = await Dataset.open(
+        storage_client=MemoryStorageClient(), configuration=configuration
+    )
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.1/docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py

@@ -0,0 +1,12 @@
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import SqlStorageClient
+
+
+async def main() -> None:
+    # Create a new instance of storage client.
+    # This will create an SQLite database file crawlee.db or created tables in your
+    # database if you pass `connection_string` or `engine`
+    # Use the context manager to ensure that connections are properly cleaned up.
+    async with SqlStorageClient() as storage_client:
+        # And pass it to the crawler.
+        crawler = ParselCrawler(storage_client=storage_client)

crawlee-1.0.1/docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py

@@ -0,0 +1,33 @@
+from sqlalchemy.ext.asyncio import create_async_engine
+
+from crawlee.configuration import Configuration
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import SqlStorageClient
+
+
+async def main() -> None:
+    # Create a new instance of storage client.
+    # On first run, also creates tables in your PostgreSQL database.
+    # Use the context manager to ensure that connections are properly cleaned up.
+    async with SqlStorageClient(
+        # Create an `engine` with the desired configuration
+        engine=create_async_engine(
+            'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres',
+            future=True,
+            pool_size=5,
+            max_overflow=10,
+            pool_recycle=3600,
+            pool_pre_ping=True,
+            echo=False,
+        )
+    ) as storage_client:
+        # Create a configuration with custom settings.
+        configuration = Configuration(
+            purge_on_start=False,
+        )
+
+        # And pass them to the crawler.
+        crawler = ParselCrawler(
+            storage_client=storage_client,
+            configuration=configuration,
+        )

crawlee-1.0.1/docs/guides/code_examples/storages/opening.py

@@ -0,0 +1,19 @@
+import asyncio
+
+from crawlee.storages import Dataset
+
+
+async def main() -> None:
+    # Named storage (persists across runs)
+    dataset_named = await Dataset.open(name='my-persistent-dataset')
+
+    # Unnamed storage with alias (purged on start)
+    dataset_unnamed = await Dataset.open(alias='temporary-results')
+
+    # Default unnamed storage (both are equivalent and purged on start)
+    dataset_default = await Dataset.open()
+    dataset_default = await Dataset.open(alias='default')
+
+
+if __name__ == '__main__':
+    asyncio.run(main())