crawlee 1.1.1b1__py3-none-any.whl → 1.2.1b7__py3-none-any.whl

crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -3,19 +3,25 @@ from __future__ import annotations
 import asyncio
 import logging
 import warnings
+from datetime import timedelta
 from functools import partial
 from typing import TYPE_CHECKING, Any, Generic, Literal
 
+import playwright.async_api
 from more_itertools import partition
 from pydantic import ValidationError
 from typing_extensions import NotRequired, TypedDict, TypeVar
 
 from crawlee import service_locator
-from crawlee._request import Request, RequestOptions
-from crawlee._types import ConcurrencySettings
+from crawlee._request import Request, RequestOptions, RequestState
+from crawlee._types import (
+    BasicCrawlingContext,
+    ConcurrencySettings,
+)
 from crawlee._utils.blocked import RETRY_CSS_SELECTORS
 from crawlee._utils.docs import docs_group
 from crawlee._utils.robots import RobotsTxtFile
+from crawlee._utils.time import SharedTimeout
 from crawlee._utils.urls import to_absolute_url_iterator
 from crawlee.browsers import BrowserPool
 from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
@@ -29,6 +35,7 @@ from crawlee.statistics import StatisticsState
 from ._playwright_crawling_context import PlaywrightCrawlingContext
 from ._playwright_http_client import PlaywrightHttpClient, browser_page_context
 from ._playwright_pre_nav_crawling_context import PlaywrightPreNavCrawlingContext
+from ._types import GotoOptions
 from ._utils import block_requests, infinite_scroll
 
 TCrawlingContext = TypeVar('TCrawlingContext', bound=PlaywrightCrawlingContext)
@@ -44,7 +51,6 @@ if TYPE_CHECKING:
 
     from crawlee import RequestTransformAction
     from crawlee._types import (
-        BasicCrawlingContext,
         EnqueueLinksKwargs,
         ExtractLinksFunction,
         HttpHeaders,
@@ -103,9 +109,11 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
         user_data_dir: str | Path | None = None,
         browser_launch_options: Mapping[str, Any] | None = None,
         browser_new_context_options: Mapping[str, Any] | None = None,
+        goto_options: GotoOptions | None = None,
         fingerprint_generator: FingerprintGenerator | None | Literal['default'] = 'default',
         headless: bool | None = None,
         use_incognito_pages: bool | None = None,
+        navigation_timeout: timedelta | None = None,
         **kwargs: Unpack[BasicCrawlerOptions[PlaywrightCrawlingContext, StatisticsState]],
     ) -> None:
         """Initialize a new instance.
@@ -134,12 +142,18 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             use_incognito_pages: By default pages share the same browser context. If set to True each page uses its
                 own context that is destroyed once the page is closed or crashes.
                 This option should not be used if `browser_pool` is provided.
+            navigation_timeout: Timeout for navigation (the process between opening a Playwright page and calling
+                the request handler)
+            goto_options: Additional options to pass to Playwright's `Page.goto()` method. The `timeout` option is
+                not supported, use `navigation_timeout` instead.
             kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`.
         """
         configuration = kwargs.pop('configuration', None)
         if configuration is not None:
             service_locator.set_configuration(configuration)
 
+        self._shared_navigation_timeouts: dict[int, SharedTimeout] = {}
+
         if browser_pool:
             # Raise an exception if browser_pool is provided together with other browser-related arguments.
             if any(
@@ -202,6 +216,9 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
         if 'concurrency_settings' not in kwargs or kwargs['concurrency_settings'] is None:
             kwargs['concurrency_settings'] = ConcurrencySettings(desired_concurrency=1)
 
+        self._navigation_timeout = navigation_timeout or timedelta(minutes=1)
+        self._goto_options = goto_options or GotoOptions()
+
         super().__init__(**kwargs)
 
     async def _open_page(
@@ -226,12 +243,21 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             log=context.log,
             page=crawlee_page.page,
             block_requests=partial(block_requests, page=crawlee_page.page),
+            goto_options=GotoOptions(**self._goto_options),
         )
 
-        async with browser_page_context(crawlee_page.page):
-            for hook in self._pre_navigation_hooks:
-                await hook(pre_navigation_context)
-        yield pre_navigation_context
+        context_id = id(pre_navigation_context)
+        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+
+        try:
+            async with browser_page_context(crawlee_page.page):
+                for hook in self._pre_navigation_hooks:
+                    async with self._shared_navigation_timeouts[context_id]:
+                        await hook(pre_navigation_context)
+
+            yield pre_navigation_context
+        finally:
+            self._shared_navigation_timeouts.pop(context_id, None)
 
     def _prepare_request_interceptor(
         self,
@@ -266,6 +292,7 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
         Raises:
             ValueError: If the browser pool is not initialized.
             SessionError: If the URL cannot be loaded by the browser.
+            TimeoutError: If navigation does not succeed within the navigation timeout.
 
         Yields:
             The enhanced crawling context with the Playwright-specific features (page, response, enqueue_links,
@@ -297,7 +324,14 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
                 # Set route_handler only for current request
                 await context.page.route(context.request.url, route_handler)
 
-            response = await context.page.goto(context.request.url)
+            try:
+                async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+                    response = await context.page.goto(
+                        context.request.url, timeout=remaining_timeout.total_seconds() * 1000, **context.goto_options
+                    )
+                context.request.state = RequestState.AFTER_NAV
+            except playwright.async_api.TimeoutError as exc:
+                raise asyncio.TimeoutError from exc
 
             if response is None:
                 raise SessionError(f'Failed to load the URL: {context.request.url}')
@@ -324,6 +358,7 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
                     extract_links=extract_links,
                     enqueue_links=self._create_enqueue_links_function(context, extract_links),
                     block_requests=partial(block_requests, page=context.page),
+                    goto_options=context.goto_options,
                 )
 
             if context.session:
@@ -364,14 +399,18 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             robots_txt_file = await self._get_robots_txt_file_for_url(context.request.url)
 
             kwargs.setdefault('strategy', 'same-hostname')
+            strategy = kwargs.get('strategy', 'same-hostname')
 
             elements = await context.page.query_selector_all(selector)
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-            links_iterator = to_absolute_url_iterator(
-                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
-            )
+
+            # Get base URL from <base> tag if present
+            extracted_base_url = await context.page.evaluate('document.baseURI')
+            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
+
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)
@@ -379,17 +418,19 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
                 skipped = iter([])
 
             for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs):
-                request_option = RequestOptions({'url': url, 'user_data': {**base_user_data}, 'label': label})
+                request_options = RequestOptions(
+                    url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy
+                )
 
                 if transform_request_function:
-                    transform_request_option = transform_request_function(request_option)
-                    if transform_request_option == 'skip':
+                    transform_request_options = transform_request_function(request_options)
+                    if transform_request_options == 'skip':
                         continue
-                    if transform_request_option != 'unchanged':
-                        request_option = transform_request_option
+                    if transform_request_options != 'unchanged':
+                        request_options = transform_request_options
 
                 try:
-                    request = Request.from_url(**request_option)
+                    request = Request.from_url(**request_options)
                 except ValidationError as exc:
                     context.log.debug(
                         f'Skipping URL "{url}" due to invalid format: {exc}. '

crawlee/crawlers/_playwright/_playwright_http_client.py

@@ -59,6 +59,7 @@ class PlaywrightHttpClient(HttpClient):
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
         statistics: Statistics | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         raise NotImplementedError('The `crawl` method should not be used for `PlaywrightHttpClient`')
 
@@ -72,6 +73,7 @@ class PlaywrightHttpClient(HttpClient):
         payload: HttpPayload | None = None,
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpResponse:
         # `proxy_info` are not used because `APIRequestContext` inherits the proxy from `BrowserContext`
         # TODO: Use `session` to restore all the fingerprint headers according to the `BrowserContext`, after resolved
@@ -87,7 +89,11 @@ class PlaywrightHttpClient(HttpClient):
 
         # Proxies appropriate to the browser context are used
         response = await browser_context.request.fetch(
-            url_or_request=url, method=method.lower(), headers=dict(headers) if headers else None, data=payload
+            url_or_request=url,
+            method=method.lower(),
+            headers=dict(headers) if headers else None,
+            data=payload,
+            timeout=timeout.total_seconds() if timeout else None,
         )
 
         return await PlaywrightHttpResponse.from_playwright_response(response, protocol='')

crawlee/crawlers/_playwright/_playwright_pre_nav_crawling_context.py

@@ -9,7 +9,7 @@ from crawlee._utils.docs import docs_group
 if TYPE_CHECKING:
     from playwright.async_api import Page
 
-    from ._types import BlockRequestsFunction
+    from ._types import BlockRequestsFunction, GotoOptions
 
 
 @dataclass(frozen=True)
@@ -26,6 +26,9 @@ class PlaywrightPreNavCrawlingContext(BasicCrawlingContext):
     block_requests: BlockRequestsFunction
     """Blocks network requests matching specified URL patterns."""
 
+    goto_options: GotoOptions
+    """Additional options to pass to Playwright's `Page.goto()` method. The `timeout` option is not supported."""
+
     async def get_snapshot(self) -> PageSnapshot:
         """Get snapshot of crawled page."""
         html = None

crawlee/crawlers/_playwright/_types.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Protocol
+from typing import TYPE_CHECKING, Literal, Protocol, TypedDict
 
 from crawlee import HttpHeaders
 from crawlee._utils.docs import docs_group
@@ -10,7 +10,7 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
 
     from playwright.async_api import APIResponse, Response
-    from typing_extensions import Self
+    from typing_extensions import NotRequired, Self
 
 
 @docs_group('Functions')
@@ -58,3 +58,13 @@ class PlaywrightHttpResponse:
         _content = await response.body()
 
         return cls(http_version=http_version, status_code=status_code, headers=headers, _content=_content)
+
+
+class GotoOptions(TypedDict):
+    """Keyword arguments for Playwright's `Page.goto()` method."""
+
+    wait_until: NotRequired[Literal['domcontentloaded', 'load', 'networkidle', 'commit']]
+    """When to consider operation succeeded, defaults to 'load' event."""
+
+    referer: NotRequired[str]
+    """Referer header value."""

crawlee/errors.py

@@ -29,6 +29,10 @@ class UserDefinedErrorHandlerError(Exception):
     """Wraps an exception thrown from an user-defined error handler."""
 
 
+class UserHandlerTimeoutError(UserDefinedErrorHandlerError):
+    """Raised when a router fails due to user raised timeout. This is different from user-defined handler timing out."""
+
+
 @docs_group('Errors')
 class SessionError(Exception):
     """Errors of `SessionError` type will trigger a session rotation.

crawlee/events/_event_manager.py

@@ -174,11 +174,9 @@ class EventManager:
             # to avoid blocking the event loop
             coro = (
                 listener(*bound_args.args, **bound_args.kwargs)
-                if asyncio.iscoroutinefunction(listener)
+                if inspect.iscoroutinefunction(listener)
                 else asyncio.to_thread(cast('Callable[..., None]', listener), *bound_args.args, **bound_args.kwargs)
             )
-            # Note: use `asyncio.iscoroutinefunction` rather then `inspect.iscoroutinefunction` since it works with
-            # unittests.mock.AsyncMock. See https://github.com/python/cpython/issues/84753.
 
             listener_task = asyncio.create_task(coro, name=f'Task-{event.value}-{listener.__name__}')
             self._listener_tasks.add(listener_task)

crawlee/http_clients/_base.py

@@ -104,6 +104,7 @@ class HttpClient(ABC):
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
         statistics: Statistics | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         """Perform the crawling for a given request.
 
@@ -114,6 +115,7 @@ class HttpClient(ABC):
             session: The session associated with the request.
             proxy_info: The information about the proxy to be used.
             statistics: The statistics object to register status codes.
+            timeout: Maximum time allowed to process the request.
 
         Raises:
             ProxyError: Raised if a proxy-related error occurs.
@@ -132,6 +134,7 @@ class HttpClient(ABC):
         payload: HttpPayload | None = None,
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpResponse:
         """Send an HTTP request via the client.
 
@@ -144,6 +147,7 @@ class HttpClient(ABC):
             payload: The data to be sent as the request body.
             session: The session associated with the request.
             proxy_info: The information about the proxy to be used.
+            timeout: Maximum time allowed to process the request.
 
         Raises:
             ProxyError: Raised if a proxy-related error occurs.

crawlee/http_clients/_curl_impersonate.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from typing import TYPE_CHECKING, Any
 
@@ -10,6 +11,7 @@ from curl_cffi.requests.cookies import Cookies as CurlCookies
 from curl_cffi.requests.cookies import CurlMorsel
 from curl_cffi.requests.exceptions import ProxyError as CurlProxyError
 from curl_cffi.requests.exceptions import RequestException as CurlRequestError
+from curl_cffi.requests.exceptions import Timeout
 from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME
 from typing_extensions import override
 
@@ -147,6 +149,7 @@ class CurlImpersonateHttpClient(HttpClient):
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
         statistics: Statistics | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         client = self._get_client(proxy_info.url if proxy_info else None)
 
@@ -157,7 +160,10 @@ class CurlImpersonateHttpClient(HttpClient):
                 headers=request.headers,
                 data=request.payload,
                 cookies=session.cookies.jar if session else None,
+                timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -186,6 +192,7 @@ class CurlImpersonateHttpClient(HttpClient):
         payload: HttpPayload | None = None,
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpResponse:
         if isinstance(headers, dict) or headers is None:
             headers = HttpHeaders(headers or {})
@@ -200,7 +207,10 @@ class CurlImpersonateHttpClient(HttpClient):
                 headers=dict(headers) if headers else None,
                 data=payload,
                 cookies=session.cookies.jar if session else None,
+                timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -241,6 +251,8 @@ class CurlImpersonateHttpClient(HttpClient):
                 stream=True,
                 timeout=timeout.total_seconds() if timeout else None,
             )
+        except Timeout as exc:
+            raise asyncio.TimeoutError from exc
         except CurlRequestError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc

crawlee/http_clients/_httpx.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from contextlib import asynccontextmanager
 from logging import getLogger
 from typing import TYPE_CHECKING, Any, cast
@@ -146,6 +147,7 @@ class HttpxHttpClient(HttpClient):
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
         statistics: Statistics | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         client = self._get_client(proxy_info.url if proxy_info else None)
         headers = self._combine_headers(request.headers)
@@ -157,10 +159,13 @@ class HttpxHttpClient(HttpClient):
             content=request.payload,
             cookies=session.cookies.jar if session else None,
             extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
+            timeout=timeout.total_seconds() if timeout is not None else httpx.USE_CLIENT_DEFAULT,
         )
 
         try:
             response = await client.send(http_request)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except httpx.TransportError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -185,6 +190,7 @@ class HttpxHttpClient(HttpClient):
         payload: HttpPayload | None = None,
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpResponse:
         client = self._get_client(proxy_info.url if proxy_info else None)
 
@@ -195,10 +201,13 @@ class HttpxHttpClient(HttpClient):
             headers=headers,
             payload=payload,
             session=session,
+            timeout=httpx.Timeout(timeout.total_seconds()) if timeout is not None else None,
         )
 
         try:
             response = await client.send(http_request)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except httpx.TransportError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -228,10 +237,13 @@ class HttpxHttpClient(HttpClient):
             headers=headers,
             payload=payload,
             session=session,
-            timeout=timeout,
+            timeout=httpx.Timeout(None, connect=timeout.total_seconds()) if timeout else None,
         )
 
-        response = await client.send(http_request, stream=True)
+        try:
+            response = await client.send(http_request, stream=True)
+        except httpx.TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
 
         try:
             yield _HttpxResponse(response)
@@ -246,7 +258,7 @@ class HttpxHttpClient(HttpClient):
         headers: HttpHeaders | dict[str, str] | None,
         payload: HttpPayload | None,
         session: Session | None = None,
-        timeout: timedelta | None = None,
+        timeout: httpx.Timeout | None = None,
     ) -> httpx.Request:
         """Build an `httpx.Request` using the provided parameters."""
         if isinstance(headers, dict) or headers is None:
@@ -254,15 +266,13 @@ class HttpxHttpClient(HttpClient):
 
         headers = self._combine_headers(headers)
 
-        httpx_timeout = httpx.Timeout(None, connect=timeout.total_seconds()) if timeout else None
-
         return client.build_request(
             url=url,
             method=method,
             headers=dict(headers) if headers else None,
             content=payload,
             extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
-            timeout=httpx_timeout,
+            timeout=timeout if timeout else httpx.USE_CLIENT_DEFAULT,
         )
 
     def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:

crawlee/http_clients/_impit.py

@@ -6,7 +6,7 @@ from logging import getLogger
 from typing import TYPE_CHECKING, Any, TypedDict
 
 from cachetools import LRUCache
-from impit import AsyncClient, Browser, HTTPError, Response, TransportError
+from impit import AsyncClient, Browser, HTTPError, Response, TimeoutException, TransportError
 from impit import ProxyError as ImpitProxyError
 from typing_extensions import override
 
@@ -125,6 +125,7 @@ class ImpitHttpClient(HttpClient):
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
         statistics: Statistics | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None)
 
@@ -134,7 +135,10 @@ class ImpitHttpClient(HttpClient):
                 method=request.method,
                 content=request.payload,
                 headers=dict(request.headers) if request.headers else None,
+                timeout=timeout.total_seconds() if timeout else None,
             )
+        except TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except (TransportError, HTTPError) as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -157,6 +161,7 @@ class ImpitHttpClient(HttpClient):
         payload: HttpPayload | None = None,
         session: Session | None = None,
         proxy_info: ProxyInfo | None = None,
+        timeout: timedelta | None = None,
     ) -> HttpResponse:
         if isinstance(headers, dict) or headers is None:
             headers = HttpHeaders(headers or {})
@@ -165,8 +170,14 @@ class ImpitHttpClient(HttpClient):
 
         try:
             response = await client.request(
-                method=method, url=url, content=payload, headers=dict(headers) if headers else None
+                method=method,
+                url=url,
+                content=payload,
+                headers=dict(headers) if headers else None,
+                timeout=timeout.total_seconds() if timeout else None,
             )
+        except TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
         except (TransportError, HTTPError) as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -189,14 +200,18 @@ class ImpitHttpClient(HttpClient):
     ) -> AsyncGenerator[HttpResponse]:
         client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None)
 
-        response = await client.request(
-            method=method,
-            url=url,
-            content=payload,
-            headers=dict(headers) if headers else None,
-            timeout=timeout.total_seconds() if timeout else None,
-            stream=True,
-        )
+        try:
+            response = await client.request(
+                method=method,
+                url=url,
+                content=payload,
+                headers=dict(headers) if headers else None,
+                timeout=timeout.total_seconds() if timeout else None,
+                stream=True,
+            )
+        except TimeoutException as exc:
+            raise asyncio.TimeoutError from exc
+
         try:
             yield _ImpitResponse(response)
         finally:

crawlee/router.py

@@ -1,13 +1,17 @@
 from __future__ import annotations
 
+import asyncio
 from collections.abc import Awaitable, Callable
 from typing import Generic, TypeVar
 
+from crawlee._request import RequestState
 from crawlee._types import BasicCrawlingContext
 from crawlee._utils.docs import docs_group
 
 __all__ = ['Router']
 
+from crawlee.errors import UserHandlerTimeoutError
+
 TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext)
 RequestHandler = Callable[[TCrawlingContext], Awaitable[None]]
 
@@ -89,13 +93,19 @@ class Router(Generic[TCrawlingContext]):
 
     async def __call__(self, context: TCrawlingContext) -> None:
         """Invoke a request handler that matches the request label (or the default)."""
+        context.request.state = RequestState.REQUEST_HANDLER
         if context.request.label is None or context.request.label not in self._handlers_by_label:
             if self._default_handler is None:
                 raise RuntimeError(
                     f'No handler matches label `{context.request.label}` and no default handler is configured'
                 )
 
-            return await self._default_handler(context)
+            user_defined_handler = self._default_handler
+        else:
+            user_defined_handler = self._handlers_by_label[context.request.label]
 
-        handler = self._handlers_by_label[context.request.label]
-        return await handler(context)
+        try:
+            return await user_defined_handler(context)
+        except asyncio.TimeoutError as e:
+            # Timeout in handler, but not timeout of handler.
+            raise UserHandlerTimeoutError('Timeout raised by user defined handler') from e

crawlee/storage_clients/_file_system/_dataset_client.py

@@ -134,7 +134,7 @@ class FileSystemDatasetClient(DatasetClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = DatasetMetadata(**file_content)
@@ -163,7 +163,7 @@ class FileSystemDatasetClient(DatasetClient):
 
             # If the dataset directory exists, reconstruct the client from the metadata file.
             if path_to_dataset.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:

crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -133,7 +133,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = KeyValueStoreMetadata(**file_content)
@@ -162,7 +162,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.
             if path_to_kvs.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:
@@ -239,7 +239,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
         # Read the metadata file
         async with self._lock:
             try:
-                file = await asyncio.to_thread(open, record_metadata_filepath)
+                file = await asyncio.to_thread(open, record_metadata_filepath, 'r', encoding='utf-8')
             except FileNotFoundError:
                 logger.warning(f'Metadata file disappeared for key "{key}", aborting get_value')
                 return None