crawlee 1.0.5b20__tar.gz → 1.0.5b22__tar.gz

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/CHANGELOG.md

@@ -10,12 +10,14 @@ All notable changes to this project will be documented in this file.
 - Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
 - Add `RedisStorageClient` based on Redis v8.0+ ([#1406](https://github.com/apify/crawlee-python/pull/1406)) ([d08d13d](https://github.com/apify/crawlee-python/commit/d08d13d39203c24ab61fe254b0956d6744db3b5f)) by [@Mantisus](https://github.com/Mantisus)
 - Add support for Python 3.14 ([#1553](https://github.com/apify/crawlee-python/pull/1553)) ([89e9130](https://github.com/apify/crawlee-python/commit/89e9130cabee0fbc974b29c26483b7fa0edf627c)) by [@Mantisus](https://github.com/Mantisus)
+- Add `transform_request_function` parameter for `SitemapRequestLoader` ([#1525](https://github.com/apify/crawlee-python/pull/1525)) ([dc90127](https://github.com/apify/crawlee-python/commit/dc901271849b239ba2a947e8ebff8e1815e8c4fb)) by [@Mantisus](https://github.com/Mantisus)
 
 ### 🐛 Bug Fixes
 
 - Improve indexing of the `request_queue_records` table for `SqlRequestQueueClient` ([#1527](https://github.com/apify/crawlee-python/pull/1527)) ([6509534](https://github.com/apify/crawlee-python/commit/65095346a9d8b703b10c91e0510154c3c48a4176)) by [@Mantisus](https://github.com/Mantisus), closes [#1526](https://github.com/apify/crawlee-python/issues/1526)
 - Improve error handling for `RobotsTxtFile.load` ([#1524](https://github.com/apify/crawlee-python/pull/1524)) ([596a311](https://github.com/apify/crawlee-python/commit/596a31184914a254b3e7a81fd2f48ea8eda7db49)) by [@Mantisus](https://github.com/Mantisus)
 - Fix `crawler_runtime` not being updated during run and only in the end ([#1540](https://github.com/apify/crawlee-python/pull/1540)) ([0d6c3f6](https://github.com/apify/crawlee-python/commit/0d6c3f6d3337ddb6cab4873747c28cf95605d550)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1541](https://github.com/apify/crawlee-python/issues/1541)
+- Ensure persist state event emission when exiting `EventManager` context ([#1562](https://github.com/apify/crawlee-python/pull/1562)) ([6a44f17](https://github.com/apify/crawlee-python/commit/6a44f172600cbcacebab899082d6efc9105c4e03)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1560](https://github.com/apify/crawlee-python/issues/1560)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b20
+Version: 1.0.5b22
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

crawlee-1.0.5b22/docs/examples/code_examples/using_sitemap_request_loader.py

@@ -0,0 +1,101 @@
+import asyncio
+from collections.abc import Callable
+
+from yarl import URL
+
+from crawlee import RequestOptions, RequestTransformAction
+from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
+from crawlee.http_clients import ImpitHttpClient
+from crawlee.request_loaders import SitemapRequestLoader
+
+
+# Create a transform_request_function that maps request options based on the host in
+# the URL
+def create_transform_request(
+    data_mapper: dict[str, dict],
+) -> Callable[[RequestOptions], RequestOptions | RequestTransformAction]:
+    def transform_request(
+        request_options: RequestOptions,
+    ) -> RequestOptions | RequestTransformAction:
+        # According to the Sitemap protocol, all URLs in a Sitemap must be from a single
+        # host.
+        request_host = URL(request_options['url']).host
+
+        if request_host and (mapping_data := data_mapper.get(request_host)):
+            # Set properties from the mapping data
+            if 'label' in mapping_data:
+                request_options['label'] = mapping_data['label']
+            if 'user_data' in mapping_data:
+                request_options['user_data'] = mapping_data['user_data']
+
+            return request_options
+
+        return 'unchanged'
+
+    return transform_request
+
+
+async def main() -> None:
+    # Prepare data mapping for hosts
+    apify_host = URL('https://apify.com/sitemap.xml').host
+    crawlee_host = URL('https://crawlee.dev/sitemap.xml').host
+
+    if not apify_host or not crawlee_host:
+        raise ValueError('Unable to extract host from URLs')
+
+    data_map = {
+        apify_host: {
+            'label': 'apify',
+            'user_data': {'source': 'apify'},
+        },
+        crawlee_host: {
+            'label': 'crawlee',
+            'user_data': {'source': 'crawlee'},
+        },
+    }
+
+    # Initialize the SitemapRequestLoader with the transform function
+    async with SitemapRequestLoader(
+        # Set the sitemap URLs and the HTTP client
+        sitemap_urls=['https://crawlee.dev/sitemap.xml', 'https://apify.com/sitemap.xml'],
+        http_client=ImpitHttpClient(),
+        transform_request_function=create_transform_request(data_map),
+    ) as sitemap_loader:
+        # Convert the sitemap loader to a request manager
+        request_manager = await sitemap_loader.to_tandem()
+
+        # Create and configure the crawler
+        crawler = BeautifulSoupCrawler(
+            request_manager=request_manager,
+            max_requests_per_crawl=10,
+        )
+
+        # Create default handler for requests without a specific label
+        @crawler.router.default_handler
+        async def handler(context: BeautifulSoupCrawlingContext) -> None:
+            source = context.request.user_data.get('source', 'unknown')
+            context.log.info(
+                f'Processing request: {context.request.url} from source: {source}'
+            )
+
+        # Create handler for requests labeled 'apify'
+        @crawler.router.handler('apify')
+        async def apify_handler(context: BeautifulSoupCrawlingContext) -> None:
+            source = context.request.user_data.get('source', 'unknown')
+            context.log.info(
+                f'Apify handler processing: {context.request.url} from source: {source}'
+            )
+
+        # Create handler for requests labeled 'crawlee'
+        @crawler.router.handler('crawlee')
+        async def crawlee_handler(context: BeautifulSoupCrawlingContext) -> None:
+            source = context.request.user_data.get('source', 'unknown')
+            context.log.info(
+                f'Crawlee handler processing: {context.request.url} from source: {source}'
+            )
+
+        await crawler.run()
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

crawlee-1.0.5b22/docs/examples/using_sitemap_request_loader.mdx

@@ -0,0 +1,22 @@
+---
+id: using-sitemap-request-loader
+title: Using sitemap request loader
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+
+import SitemapRequestLoaderExample from '!!raw-loader!roa-loader!./code_examples/using_sitemap_request_loader.py';
+
+This example demonstrates how to use <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> to crawl websites that provide `sitemap.xml` files following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html). The <ApiLink to="class/SitemapRequestLoader">`SitemapRequestLoader`</ApiLink> processes sitemaps in a streaming fashion without loading them entirely into memory, making it suitable for large sitemaps.
+
+The example shows how to use the `transform_request_function` parameter to configure request options based on URL patterns. This allows you to modify request properties such as labels and user data based on the source URL, enabling different handling logic for different websites or sections.
+
+The following code example implements processing of sitemaps from two different domains (Apify and Crawlee), with different labels assigned to requests based on their host. The `create_transform_request` function maps each host to the corresponding request configuration, while the crawler uses different handlers based on the assigned labels.
+
+<RunnableCodeBlock className="language-python" language="python">
+    {SitemapRequestLoaderExample}
+</RunnableCodeBlock>
+
+For more information about request loaders, see the [Request loaders guide](../guides/request-loaders).

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.5b20"
+version = "1.0.5b22"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -56,7 +56,7 @@ from crawlee.errors import (
     SessionError,
     UserDefinedErrorHandlerError,
 )
-from crawlee.events._types import Event, EventCrawlerStatusData, EventPersistStateData
+from crawlee.events._types import Event, EventCrawlerStatusData
 from crawlee.http_clients import ImpitHttpClient
 from crawlee.router import Router
 from crawlee.sessions import SessionPool
@@ -751,9 +751,6 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
 
             await self._autoscaled_pool.run()
 
-            # Emit PERSIST_STATE event when crawler is finishing to allow listeners to persist their state if needed
-            event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False))
-
     async def add_requests(
         self,
         requests: Sequence[str | Request],

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/src/crawlee/events/_event_manager.py

@@ -130,11 +130,13 @@ class EventManager:
         if not self._active:
             raise RuntimeError(f'The {self.__class__.__name__} is not active.')
 
+        # Stop persist state event periodic emission and manually emit last one to ensure latest state is saved.
+        await self._emit_persist_state_event_rec_task.stop()
+        await self._emit_persist_state_event()
         await self.wait_for_all_listeners_to_complete(timeout=self._close_timeout)
         self._event_emitter.remove_all_listeners()
         self._listener_tasks.clear()
         self._listeners_to_wrappers.clear()
-        await self._emit_persist_state_event_rec_task.stop()
         self._active = False
 
     @overload

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/src/crawlee/request_loaders/_sitemap_request_loader.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Annotated, Any
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import override
 
-from crawlee import Request
+from crawlee import Request, RequestOptions
 from crawlee._utils.docs import docs_group
 from crawlee._utils.globs import Glob
 from crawlee._utils.recoverable_state import RecoverableState
@@ -18,9 +18,10 @@ from crawlee.request_loaders._request_loader import RequestLoader
 
 if TYPE_CHECKING:
     import re
-    from collections.abc import Sequence
+    from collections.abc import Callable, Sequence
     from types import TracebackType
 
+    from crawlee import RequestTransformAction
     from crawlee.http_clients import HttpClient
     from crawlee.proxy_configuration import ProxyInfo
     from crawlee.storage_clients.models import ProcessedRequest
@@ -112,6 +113,7 @@ class SitemapRequestLoader(RequestLoader):
         exclude: list[re.Pattern[Any] | Glob] | None = None,
         max_buffer_size: int = 200,
         persist_state_key: str | None = None,
+        transform_request_function: Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None = None,
     ) -> None:
         """Initialize the sitemap request loader.
 
@@ -125,6 +127,9 @@ class SitemapRequestLoader(RequestLoader):
             persist_state_key: A key for persisting the loader's state in the KeyValueStore.
                 When provided, allows resuming from where it left off after interruption.
                 If None, no state persistence occurs.
+            transform_request_function: An optional function to transform requests
+                generated by the loader. It receives `RequestOptions` with `url` and should return either
+                modified `RequestOptions` or a `RequestTransformAction`.
         """
         self._http_client = http_client
         self._sitemap_urls = sitemap_urls
@@ -132,6 +137,7 @@ class SitemapRequestLoader(RequestLoader):
         self._exclude = exclude
         self._proxy_info = proxy_info
         self._max_buffer_size = max_buffer_size
+        self._transform_request_function = transform_request_function
 
         # Synchronization for queue operations
         self._queue_has_capacity = asyncio.Event()
@@ -313,8 +319,15 @@ class SitemapRequestLoader(RequestLoader):
 
             async with self._queue_lock:
                 url = state.url_queue.popleft()
-
-                request = Request.from_url(url)
+                request_option = RequestOptions(url=url)
+                if self._transform_request_function:
+                    transform_request_option = self._transform_request_function(request_option)
+                    if transform_request_option == 'skip':
+                        state.total_count -= 1
+                        continue
+                    if transform_request_option != 'unchanged':
+                        request_option = transform_request_option
+                request = Request.from_url(**request_option)
                 state.in_progress.add(request.url)
                 if len(state.url_queue) < self._max_buffer_size:
                     self._queue_has_capacity.set()

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/src/crawlee/statistics/_models.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import warnings
 from dataclasses import asdict, dataclass
 from datetime import datetime, timedelta, timezone
 from typing import Annotated, Any
@@ -76,7 +77,6 @@ class StatisticsState(BaseModel):
     crawler_started_at: Annotated[datetime | None, Field(alias='crawlerStartedAt')] = None
     crawler_last_started_at: Annotated[datetime | None, Field(alias='crawlerLastStartTimestamp')] = None
     crawler_finished_at: Annotated[datetime | None, Field(alias='crawlerFinishedAt')] = None
-    crawler_runtime: Annotated[timedelta_ms, Field(alias='crawlerRuntimeMillis')] = timedelta()
     errors: dict[str, Any] = Field(default_factory=dict)
     retry_errors: dict[str, Any] = Field(alias='retryErrors', default_factory=dict)
     requests_with_status_code: dict[str, int] = Field(alias='requestsWithStatusCode', default_factory=dict)
@@ -93,6 +93,37 @@ class StatisticsState(BaseModel):
         ),
     ] = {}
 
+    # Used to track the crawler runtime, that had already been persisted. This is the runtime from previous runs.
+    _runtime_offset: Annotated[timedelta, Field(exclude=True)] = timedelta()
+
+    def model_post_init(self, /, __context: Any) -> None:
+        self._runtime_offset = self.crawler_runtime or self._runtime_offset
+
+    @property
+    def crawler_runtime(self) -> timedelta:
+        if self.crawler_last_started_at:
+            finished_at = self.crawler_finished_at or datetime.now(timezone.utc)
+            return self._runtime_offset + finished_at - self.crawler_last_started_at
+        return self._runtime_offset
+
+    @crawler_runtime.setter
+    def crawler_runtime(self, value: timedelta) -> None:
+        # Setter for backwards compatibility only, the crawler_runtime is now computed_field, and cant be set manually.
+        # To be removed in v2 release https://github.com/apify/crawlee-python/issues/1567
+        warnings.warn(
+            f"Setting 'crawler_runtime' is deprecated and will be removed in a future version."
+            f' Value {value} will not be used.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+    @computed_field(alias='crawlerRuntimeMillis')
+    def crawler_runtime_for_serialization(self) -> timedelta:
+        if self.crawler_last_started_at:
+            finished_at = self.crawler_finished_at or datetime.now(timezone.utc)
+            return self._runtime_offset + finished_at - self.crawler_last_started_at
+        return self._runtime_offset
+
     @computed_field(alias='requestTotalDurationMillis', return_type=timedelta_ms)  # type: ignore[prop-decorator]
     @property
     def request_total_duration(self) -> timedelta:

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/src/crawlee/statistics/_statistics.py

@@ -110,9 +110,6 @@ class Statistics(Generic[TStatisticsState]):
         # Flag to indicate the context state.
         self._active = False
 
-        # Pre-existing runtime offset, that can be non-zero when restoring serialized state from KVS.
-        self._runtime_offset = timedelta(seconds=0)
-
     def replace_state_model(self, state_model: type[TNewStatisticsState]) -> Statistics[TNewStatisticsState]:
         """Create near copy of the `Statistics` with replaced `state_model`."""
         new_statistics: Statistics[TNewStatisticsState] = Statistics(
@@ -168,8 +165,8 @@ class Statistics(Generic[TStatisticsState]):
             raise RuntimeError(f'The {self.__class__.__name__} is already active.')
 
         await self._state.initialize()
-
-        self._runtime_offset = self.state.crawler_runtime
+        # Reset `crawler_finished_at` to indicate a new run in progress.
+        self.state.crawler_finished_at = None
 
         # Start periodic logging and let it print initial state before activation.
         self._periodic_logger.start()
@@ -200,10 +197,6 @@ class Statistics(Generic[TStatisticsState]):
         # Stop logging and deactivate the statistics to prevent further changes to crawler_runtime
         await self._periodic_logger.stop()
         self.state.crawler_finished_at = datetime.now(timezone.utc)
-        self.state.crawler_runtime = (
-            self._runtime_offset + self.state.crawler_finished_at - self.state.crawler_last_started_at
-        )
-
         self._active = False
         await self._state.teardown()
 
@@ -262,20 +255,8 @@ class Statistics(Generic[TStatisticsState]):
 
         del self._requests_in_progress[request_id_or_key]
 
-    def _update_crawler_runtime(self) -> None:
-        current_run_duration = (
-            (datetime.now(timezone.utc) - self.state.crawler_last_started_at)
-            if self.state.crawler_last_started_at
-            else timedelta()
-        )
-        self.state.crawler_runtime = current_run_duration + self._runtime_offset
-
     def calculate(self) -> FinalStatistics:
         """Calculate the current statistics."""
-        if self._active:
-            # Only update state when active. If not, just report the last known runtime.
-            self._update_crawler_runtime()
-
         total_minutes = self.state.crawler_runtime.total_seconds() / 60
         state = self._state.current_value
         serialized_state = state.model_dump(by_alias=False)

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1673,7 +1673,6 @@ def _process_run_crawler(requests: list[str], storage_dir: str) -> StatisticsSta
     return asyncio.run(_run_crawler(requests=requests, storage_dir=storage_dir))
 
 
-@pytest.mark.skip(reason='This test is flaky, see https://github.com/apify/crawlee-python/issues/1560.')
 async def test_crawler_statistics_persistence(tmp_path: Path) -> None:
     """Test that crawler statistics persist and are loaded correctly.
 

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/tests/unit/events/test_event_manager.py

@@ -5,6 +5,7 @@ import logging
 from datetime import timedelta
 from functools import update_wrapper
 from typing import TYPE_CHECKING, Any
+from unittest import mock
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -207,3 +208,14 @@ async def test_methods_raise_error_when_not_active(event_system_info_data: Event
         await event_manager.wait_for_all_listeners_to_complete()
 
         assert event_manager.active is True
+
+
+async def test_event_manager_in_context_persistence() -> None:
+    """Test that entering the `EventManager` context emits persist state event at least once."""
+    event_manager = EventManager()
+
+    with mock.patch.object(event_manager, '_emit_persist_state_event', AsyncMock()) as mocked_emit_persist_state_event:
+        async with event_manager:
+            pass
+
+    assert mocked_emit_persist_state_event.call_count >= 1

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/tests/unit/request_loaders/test_sitemap_request_loader.py

@@ -4,6 +4,7 @@ import gzip
 
 from yarl import URL
 
+from crawlee import RequestOptions, RequestTransformAction
 from crawlee.http_clients._base import HttpClient
 from crawlee.request_loaders._sitemap_request_loader import SitemapRequestLoader
 from crawlee.storages import KeyValueStore
@@ -172,3 +173,37 @@ async def test_recovery_data_persistence_for_sitemap_loading(
 
     assert item is not None
     assert item.url == next_item_in_kvs
+
+
+async def test_transform_request_function(server_url: URL, http_client: HttpClient) -> None:
+    sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
+
+    def transform_request(request_options: RequestOptions) -> RequestOptions | RequestTransformAction:
+        request_options['user_data'] = {'transformed': True}
+        return request_options
+
+    sitemap_loader = SitemapRequestLoader(
+        [str(sitemap_url)],
+        http_client=http_client,
+        transform_request_function=transform_request,
+    )
+
+    extracted_urls = set()
+
+    while not await sitemap_loader.is_finished():
+        request = await sitemap_loader.fetch_next_request()
+        assert request is not None
+        assert request.user_data.get('transformed') is True
+
+        extracted_urls.add(request.url)
+
+        await sitemap_loader.mark_request_as_handled(request)
+
+    assert len(extracted_urls) == 5
+    assert extracted_urls == {
+        'http://not-exists.com/',
+        'http://not-exists.com/catalog?item=12&desc=vacation_hawaii',
+        'http://not-exists.com/catalog?item=73&desc=vacation_new_zealand',
+        'http://not-exists.com/catalog?item=74&desc=vacation_newfoundland',
+        'http://not-exists.com/catalog?item=83&desc=vacation_usa',
+    }

{crawlee-1.0.5b20 → crawlee-1.0.5b22}/uv.lock

@@ -693,7 +693,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.5b20"
+version = "1.0.5b22"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },