crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/_types.py

@@ -2,18 +2,9 @@ from __future__ import annotations
 
 import dataclasses
 from collections.abc import Callable, Iterator, Mapping
+from copy import deepcopy
 from dataclasses import dataclass
-from typing import (
-    TYPE_CHECKING,
-    Annotated,
-    Any,
-    Literal,
-    Protocol,
-    TypedDict,
-    TypeVar,
-    cast,
-    overload,
-)
+from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol, TypedDict, TypeVar, cast, overload
 
 from pydantic import ConfigDict, Field, PlainValidator, RootModel
 
@@ -25,7 +16,7 @@ if TYPE_CHECKING:
     import re
     from collections.abc import Callable, Coroutine, Sequence
 
-    from typing_extensions import NotRequired, Required, Unpack
+    from typing_extensions import NotRequired, Required, Self, Unpack
 
     from crawlee import Glob, Request
     from crawlee._request import RequestOptions
@@ -69,13 +60,17 @@ def _normalize_headers(headers: Mapping[str, str]) -> dict[str, str]:
 class HttpHeaders(RootModel, Mapping[str, str]):
     """A dictionary-like object representing HTTP headers."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
-    root: Annotated[
-        dict[str, str],
-        PlainValidator(lambda value: _normalize_headers(value)),
-        Field(default_factory=dict),
-    ] = {}
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        root: dict[str, str] = {}
+    else:
+        root: Annotated[
+            dict[str, str],
+            PlainValidator(lambda value: _normalize_headers(value)),
+            Field(default_factory=lambda: dict[str, str]()),
+        ]
 
     def __getitem__(self, key: str) -> str:
         return self.root[key.lower()]
@@ -96,7 +91,7 @@ class HttpHeaders(RootModel, Mapping[str, str]):
         combined_headers = {**other, **self.root}
         return HttpHeaders(combined_headers)
 
-    def __iter__(self) -> Iterator[str]:  # type: ignore[override]
+    def __iter__(self) -> Iterator[str]:  # ty: ignore[invalid-method-override]
         yield from self.root
 
     def __len__(self) -> int:
@@ -110,9 +105,9 @@ class ConcurrencySettings:
     def __init__(
         self,
         min_concurrency: int = 1,
-        max_concurrency: int = 200,
+        max_concurrency: int = 100,
         max_tasks_per_minute: float = float('inf'),
-        desired_concurrency: int | None = None,
+        desired_concurrency: int = 10,
     ) -> None:
         """Initialize a new instance.
 
@@ -125,21 +120,24 @@ class ConcurrencySettings:
             desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool,
                 if there is a large enough supply of them. By default, it is `min_concurrency`.
         """
-        if desired_concurrency is not None and desired_concurrency < 1:
-            raise ValueError('desired_concurrency must be 1 or larger')
-
         if min_concurrency < 1:
             raise ValueError('min_concurrency must be 1 or larger')
 
         if max_concurrency < min_concurrency:
             raise ValueError('max_concurrency cannot be less than min_concurrency')
 
+        if desired_concurrency < min_concurrency:
+            raise ValueError('desired_concurrency cannot be less than min_concurrency')
+
+        if desired_concurrency > max_concurrency:
+            raise ValueError('desired_concurrency cannot be greater than max_concurrency')
+
         if max_tasks_per_minute <= 0:
             raise ValueError('max_tasks_per_minute must be positive')
 
         self.min_concurrency = min_concurrency
         self.max_concurrency = max_concurrency
-        self.desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency
+        self.desired_concurrency = desired_concurrency
         self.max_tasks_per_minute = max_tasks_per_minute
 
 
@@ -180,6 +178,17 @@ class AddRequestsKwargs(EnqueueLinksKwargs):
     requests: Sequence[str | Request]
     """Requests to be added to the `RequestManager`."""
 
+    rq_id: str | None
+    """ID of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias` can be provided."""
+
+    rq_name: str | None
+    """Name of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias` can be provided.
+    """
+
+    rq_alias: str | None
+    """Alias of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias` can be provided.
+    """
+
 
 class PushDataKwargs(TypedDict):
     """Keyword arguments for dataset's `push_data` method."""
@@ -189,6 +198,7 @@ class PushDataFunctionCall(PushDataKwargs):
     data: list[dict[str, Any]] | dict[str, Any]
     dataset_id: str | None
     dataset_name: str | None
+    dataset_alias: str | None
 
 
 class KeyValueStoreInterface(Protocol):
@@ -251,25 +261,46 @@ class KeyValueStoreChangeRecords:
 class RequestHandlerRunResult:
     """Record of calls to storage-related context helpers."""
 
-    def __init__(self, *, key_value_store_getter: GetKeyValueStoreFunction) -> None:
+    def __init__(
+        self,
+        *,
+        key_value_store_getter: GetKeyValueStoreFunction,
+        request: Request,
+    ) -> None:
         self._key_value_store_getter = key_value_store_getter
         self.add_requests_calls = list[AddRequestsKwargs]()
         self.push_data_calls = list[PushDataFunctionCall]()
-        self.key_value_store_changes = dict[tuple[str | None, str | None], KeyValueStoreChangeRecords]()
+        self.key_value_store_changes = dict[tuple[str | None, str | None, str | None], KeyValueStoreChangeRecords]()
+
+        # Isolated copies for handler execution
+        self._request = deepcopy(request)
+
+    @property
+    def request(self) -> Request:
+        return self._request
 
     async def add_requests(
         self,
         requests: Sequence[str | Request],
+        rq_id: str | None = None,
+        rq_name: str | None = None,
+        rq_alias: str | None = None,
         **kwargs: Unpack[EnqueueLinksKwargs],
     ) -> None:
         """Track a call to the `add_requests` context helper."""
-        self.add_requests_calls.append(AddRequestsKwargs(requests=requests, **kwargs))
+        specified_params = sum(1 for param in [rq_id, rq_name, rq_alias] if param is not None)
+        if specified_params > 1:
+            raise ValueError('Only one of `rq_id`, `rq_name` or `rq_alias` can be provided.')
+        self.add_requests_calls.append(
+            AddRequestsKwargs(requests=requests, rq_id=rq_id, rq_name=rq_name, rq_alias=rq_alias, **kwargs)
+        )
 
     async def push_data(
         self,
         data: list[dict[str, Any]] | dict[str, Any],
         dataset_id: str | None = None,
         dataset_name: str | None = None,
+        dataset_alias: str | None = None,
         **kwargs: Unpack[PushDataKwargs],
     ) -> None:
         """Track a call to the `push_data` context helper."""
@@ -278,6 +309,7 @@ class RequestHandlerRunResult:
                 data=data,
                 dataset_id=dataset_id,
                 dataset_name=dataset_name,
+                dataset_alias=dataset_alias,
                 **kwargs,
             )
         )
@@ -287,13 +319,22 @@ class RequestHandlerRunResult:
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
     ) -> KeyValueStoreInterface:
-        if (id, name) not in self.key_value_store_changes:
-            self.key_value_store_changes[id, name] = KeyValueStoreChangeRecords(
-                await self._key_value_store_getter(id=id, name=name)
+        if (id, name, alias) not in self.key_value_store_changes:
+            self.key_value_store_changes[id, name, alias] = KeyValueStoreChangeRecords(
+                await self._key_value_store_getter(id=id, name=name, alias=alias)
             )
 
-        return self.key_value_store_changes[id, name]
+        return self.key_value_store_changes[id, name, alias]
+
+    def apply_request_changes(self, target: Request) -> None:
+        """Apply tracked changes from handler copy to original request."""
+        if self.request.user_data != target.user_data:
+            target.user_data = self.request.user_data
+
+        if self.request.headers != target.headers:
+            target.headers = self.request.headers
 
 
 @docs_group('Functions')
@@ -307,12 +348,21 @@ class AddRequestsFunction(Protocol):
     def __call__(
         self,
         requests: Sequence[str | Request],
+        rq_id: str | None = None,
+        rq_name: str | None = None,
+        rq_alias: str | None = None,
         **kwargs: Unpack[EnqueueLinksKwargs],
     ) -> Coroutine[None, None, None]:
         """Call dunder method.
 
         Args:
             requests: Requests to be added to the `RequestManager`.
+            rq_id: ID of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias` can be
+                provided.
+            rq_name: Name of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias`
+                can be provided.
+            rq_alias: Alias of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias`
+                can be provided.
             **kwargs: Additional keyword arguments.
         """
 
@@ -340,12 +390,21 @@ class EnqueueLinksFunction(Protocol):
         label: str | None = None,
         user_data: dict[str, Any] | None = None,
         transform_request_function: Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None = None,
+        rq_id: str | None = None,
+        rq_name: str | None = None,
+        rq_alias: str | None = None,
         **kwargs: Unpack[EnqueueLinksKwargs],
     ) -> Coroutine[None, None, None]: ...
 
     @overload
     def __call__(
-        self, *, requests: Sequence[str | Request] | None = None, **kwargs: Unpack[EnqueueLinksKwargs]
+        self,
+        *,
+        requests: Sequence[str | Request] | None = None,
+        rq_id: str | None = None,
+        rq_name: str | None = None,
+        rq_alias: str | None = None,
+        **kwargs: Unpack[EnqueueLinksKwargs],
     ) -> Coroutine[None, None, None]: ...
 
     def __call__(
@@ -356,6 +415,9 @@ class EnqueueLinksFunction(Protocol):
         user_data: dict[str, Any] | None = None,
         transform_request_function: Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None = None,
         requests: Sequence[str | Request] | None = None,
+        rq_id: str | None = None,
+        rq_name: str | None = None,
+        rq_alias: str | None = None,
         **kwargs: Unpack[EnqueueLinksKwargs],
     ) -> Coroutine[None, None, None]:
         """Call enqueue links function.
@@ -373,6 +435,12 @@ class EnqueueLinksFunction(Protocol):
                 - `'skip'` to exclude the request from being enqueued,
                 - `'unchanged'` to use the original request options without modification.
             requests: Requests to be added to the `RequestManager`.
+            rq_id: ID of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias` can be
+                provided.
+            rq_name: Name of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias`
+                can be provided.
+            rq_alias: Alias of the `RequestQueue` to add the requests to. Only one of `rq_id`, `rq_name` or `rq_alias`
+                can be provided.
             **kwargs: Additional keyword arguments.
         """
 
@@ -424,12 +492,14 @@ class GetKeyValueStoreFunction(Protocol):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
     ) -> Coroutine[None, None, KeyValueStore]:
         """Call dunder method.
 
         Args:
             id: The ID of the `KeyValueStore` to get.
-            name: The name of the `KeyValueStore` to get.
+            name: The name of the `KeyValueStore` to get (global scope, named storage).
+            alias: The alias of the `KeyValueStore` to get (run scope, unnamed storage).
         """
 
 
@@ -444,12 +514,14 @@ class GetKeyValueStoreFromRequestHandlerFunction(Protocol):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
     ) -> Coroutine[None, None, KeyValueStoreInterface]:
         """Call dunder method.
 
         Args:
             id: The ID of the `KeyValueStore` to get.
-            name: The name of the `KeyValueStore` to get.
+            name: The name of the `KeyValueStore` to get (global scope, named storage).
+            alias: The alias of the `KeyValueStore` to get (run scope, unnamed storage).
         """
 
 
@@ -466,6 +538,7 @@ class PushDataFunction(Protocol):
         data: list[dict[str, Any]] | dict[str, Any],
         dataset_id: str | None = None,
         dataset_name: str | None = None,
+        dataset_alias: str | None = None,
         **kwargs: Unpack[PushDataKwargs],
     ) -> Coroutine[None, None, None]:
         """Call dunder method.
@@ -473,7 +546,8 @@ class PushDataFunction(Protocol):
         Args:
             data: The data to push to the `Dataset`.
             dataset_id: The ID of the `Dataset` to push the data to.
-            dataset_name: The name of the `Dataset` to push the data to.
+            dataset_name: The name of the `Dataset` to push the data to (global scope, named storage).
+            dataset_alias: The alias of the `Dataset` to push the data to (run scope, unnamed storage).
             **kwargs: Additional keyword arguments.
         """
 
@@ -590,6 +664,24 @@ class BasicCrawlingContext:
         """Return hash of the context. Each context is considered unique."""
         return id(self)
 
+    def create_modified_copy(
+        self,
+        push_data: PushDataFunction | None = None,
+        add_requests: AddRequestsFunction | None = None,
+        get_key_value_store: GetKeyValueStoreFromRequestHandlerFunction | None = None,
+    ) -> Self:
+        """Create a modified copy of the crawling context with specified changes."""
+        modifications = dict[str, Any]()
+
+        if push_data is not None:
+            modifications['push_data'] = push_data
+        if add_requests is not None:
+            modifications['add_requests'] = add_requests
+        if get_key_value_store is not None:
+            modifications['get_key_value_store'] = get_key_value_store
+
+        return dataclasses.replace(self, **modifications)
+
 
 class GetDataKwargs(TypedDict):
     """Keyword arguments for dataset's `get_data` method."""

crawlee/_utils/context.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-import asyncio
+import inspect
 from collections.abc import Callable
 from functools import wraps
-from typing import Any, TypeVar
+from typing import Any, TypeVar, cast
 
 T = TypeVar('T', bound=Callable[..., Any])
 
@@ -44,4 +44,4 @@ def ensure_context(method: T) -> T:
 
         return await method(self, *args, **kwargs)
 
-    return async_wrapper if asyncio.iscoroutinefunction(method) else sync_wrapper  # type: ignore[return-value]
+    return cast('T', async_wrapper if inspect.iscoroutinefunction(method) else sync_wrapper)

crawlee/_utils/file.py

@@ -163,7 +163,14 @@ async def export_csv_to_stream(
     dst: TextIO,
     **kwargs: Unpack[ExportDataCsvKwargs],
 ) -> None:
-    writer = csv.writer(dst, **kwargs)  # type: ignore[arg-type]
+    # Set lineterminator to '\n' if not explicitly provided. This prevents double line endings on Windows.
+    # The csv.writer default is '\r\n', which when written to a file in text mode on Windows gets converted
+    # to '\r\r\n' due to newline translation. By using '\n', we let the platform handle the line ending
+    # conversion: '\n' stays as '\n' on Unix, and becomes '\r\n' on Windows.
+    if 'lineterminator' not in kwargs:
+        kwargs['lineterminator'] = '\n'
+
+    writer = csv.writer(dst, **kwargs)
     write_header = True
 
     # Iterate over the dataset and write to CSV.

crawlee/_utils/globs.py

@@ -33,12 +33,12 @@ def _translate(
 
     HACK: This function is copied from CPython stdlib source. It will be released in Python 3.13 as `glob.translate`
     """
-    if not seps:
-        seps = (os.path.sep, os.path.altsep) if os.path.altsep else os.path.sep
+    _seps = ((os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)) if seps is None else seps
 
-    escaped_seps = ''.join(map(re.escape, seps))
-    any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
+    escaped_seps = ''.join(map(re.escape, _seps))
+    any_sep = f'[{escaped_seps}]' if len(_seps) > 1 else escaped_seps
     not_sep = f'[^{escaped_seps}]'
+
     if include_hidden:
         one_last_segment = f'{not_sep}+'
         one_segment = f'{one_last_segment}{any_sep}'

crawlee/_utils/raise_if_too_many_kwargs.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+
+def raise_if_too_many_kwargs(max_kwargs: int = 1, **kwargs: Any) -> None:
+    """Raise ValueError if there are more non-None kwargs then max_kwargs."""
+    none_kwargs_names = [f'"{kwarg_name}"' for kwarg_name, value in kwargs.items() if value is not None]
+    if len(none_kwargs_names) > max_kwargs:
+        all_kwargs_names = [f'"{kwarg_name}"' for kwarg_name in kwargs]
+        raise ValueError(
+            f'Only one of {", ".join(all_kwargs_names)} can be specified, but following arguments were '
+            f'specified: {", ".join(none_kwargs_names)}.'
+        )

crawlee/_utils/recoverable_state.py

@@ -4,12 +4,14 @@ from typing import TYPE_CHECKING, Generic, Literal, TypeVar
 
 from pydantic import BaseModel
 
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.events._types import Event, EventPersistStateData
 
 if TYPE_CHECKING:
     import logging
+    from collections.abc import Callable, Coroutine
 
-    from crawlee.storages._key_value_store import KeyValueStore
+    from crawlee.storages import KeyValueStore
 
 TStateModel = TypeVar('TStateModel', bound=BaseModel)
 
@@ -37,6 +39,7 @@ class RecoverableState(Generic[TStateModel]):
         persistence_enabled: Literal[True, False, 'explicit_only'] = False,
         persist_state_kvs_name: str | None = None,
         persist_state_kvs_id: str | None = None,
+        persist_state_kvs_factory: Callable[[], Coroutine[None, None, KeyValueStore]] | None = None,
         logger: logging.Logger,
     ) -> None:
         """Initialize a new recoverable state object.
@@ -51,16 +54,40 @@ class RecoverableState(Generic[TStateModel]):
                 If neither a name nor and id are supplied, the default store will be used.
             persist_state_kvs_id: The identifier of the KeyValueStore to use for persistence.
                 If neither a name nor and id are supplied, the default store will be used.
+            persist_state_kvs_factory: Factory that can be awaited to create KeyValueStore to use for persistence. If
+                not provided, a system-wide KeyValueStore will be used, based on service locator configuration.
             logger: A logger instance for logging operations related to state persistence
         """
+        raise_if_too_many_kwargs(
+            persist_state_kvs_name=persist_state_kvs_name,
+            persist_state_kvs_id=persist_state_kvs_id,
+            persist_state_kvs_factory=persist_state_kvs_factory,
+        )
+        if not persist_state_kvs_factory:
+            logger.debug(
+                'No explicit key_value_store set for recoverable state. Recovery will use a system-wide KeyValueStore '
+                'based on service_locator configuration, potentially calling service_locator.set_storage_client in the '
+                'process. It is recommended to initialize RecoverableState with explicit key_value_store to avoid '
+                'global side effects.'
+            )
+
         self._default_state = default_state
         self._state_type: type[TStateModel] = self._default_state.__class__
         self._state: TStateModel | None = None
         self._persistence_enabled = persistence_enabled
         self._persist_state_key = persist_state_key
-        self._persist_state_kvs_name = persist_state_kvs_name
-        self._persist_state_kvs_id = persist_state_kvs_id
-        self._key_value_store: 'KeyValueStore | None' = None  # noqa: UP037
+        if persist_state_kvs_factory is None:
+
+            async def kvs_factory() -> KeyValueStore:
+                from crawlee.storages import KeyValueStore  # noqa: PLC0415 avoid circular import
+
+                return await KeyValueStore.open(name=persist_state_kvs_name, id=persist_state_kvs_id)
+
+            self._persist_state_kvs_factory = kvs_factory
+        else:
+            self._persist_state_kvs_factory = persist_state_kvs_factory
+
+        self._key_value_store: KeyValueStore | None = None
         self._log = logger
 
     async def initialize(self) -> TStateModel:
@@ -77,11 +104,8 @@ class RecoverableState(Generic[TStateModel]):
             return self.current_value
 
         # Import here to avoid circular imports.
-        from crawlee.storages._key_value_store import KeyValueStore  # noqa: PLC0415
 
-        self._key_value_store = await KeyValueStore.open(
-            name=self._persist_state_kvs_name, id=self._persist_state_kvs_id
-        )
+        self._key_value_store = await self._persist_state_kvs_factory()
 
         await self._load_saved_state()
 

crawlee/_utils/recurring_task.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
 import asyncio
+import inspect
 from logging import getLogger
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from collections.abc import Callable
     from datetime import timedelta
+    from types import TracebackType
+
+    from typing_extensions import Self
 
 logger = getLogger(__name__)
 
@@ -21,11 +25,27 @@ class RecurringTask:
     """
 
     def __init__(self, func: Callable, delay: timedelta) -> None:
-        logger.debug(f'Calling RecurringTask.__init__(func={func.__name__}, delay={delay})...')
+        logger.debug(
+            'Calling RecurringTask.__init__(func={%s}, delay={%s})...',
+            func.__name__ if hasattr(func, '__name__') else func.__class__.__name__,
+            delay,
+        )
         self.func = func
         self.delay = delay
         self.task: asyncio.Task | None = None
 
+    async def __aenter__(self) -> Self:
+        self.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        await self.stop()
+
     async def _wrapper(self) -> None:
         """Continuously execute the provided function with the specified delay.
 
@@ -34,12 +54,16 @@ class RecurringTask:
         """
         sleep_time_secs = self.delay.total_seconds()
         while True:
-            await self.func() if asyncio.iscoroutinefunction(self.func) else self.func()
+            await self.func() if inspect.iscoroutinefunction(self.func) else self.func()
             await asyncio.sleep(sleep_time_secs)
 
     def start(self) -> None:
         """Start the recurring task execution."""
-        self.task = asyncio.create_task(self._wrapper(), name=f'Task-recurring-{self.func.__name__}')
+        name = self.func.__name__ if hasattr(self.func, '__name__') else self.func.__class__.__name__
+        self.task = asyncio.create_task(
+            self._wrapper(),
+            name=f'Task-recurring-{name}',
+        )
 
     async def stop(self) -> None:
         """Stop the recurring task execution."""

crawlee/_utils/requests.py

@@ -1,8 +1,5 @@
 from __future__ import annotations
 
-import re
-from base64 import b64encode
-from hashlib import sha256
 from logging import getLogger
 from typing import TYPE_CHECKING
 
@@ -16,29 +13,6 @@ if TYPE_CHECKING:
 logger = getLogger(__name__)
 
 
-def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> str:
-    """Generate a deterministic request ID based on a unique key.
-
-    Args:
-        unique_key: The unique key to convert into a request ID.
-        request_id_length: The length of the request ID.
-
-    Returns:
-        A URL-safe, truncated request ID based on the unique key.
-    """
-    # Encode the unique key and compute its SHA-256 hash
-    hashed_key = sha256(unique_key.encode('utf-8')).digest()
-
-    # Encode the hash in base64 and decode it to get a string
-    base64_encoded = b64encode(hashed_key).decode('utf-8')
-
-    # Remove characters that are not URL-safe ('+', '/', or '=')
-    url_safe_key = re.sub(r'(\+|\/|=)', '', base64_encoded)
-
-    # Truncate the key to the desired length
-    return url_safe_key[:request_id_length]
-
-
 def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
     """Normalize a URL.
 

crawlee/_utils/robots.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from protego import Protego
@@ -15,6 +16,9 @@ if TYPE_CHECKING:
     from crawlee.proxy_configuration import ProxyInfo
 
 
+logger = getLogger(__name__)
+
+
 class RobotsTxtFile:
     def __init__(
         self, url: str, robots: Protego, http_client: HttpClient | None = None, proxy_info: ProxyInfo | None = None
@@ -56,12 +60,20 @@ class RobotsTxtFile:
             http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file.
             proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
         """
-        response = await http_client.send_request(url, proxy_info=proxy_info)
-        body = (
-            b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else await response.read()
-        )
+        try:
+            response = await http_client.send_request(url, proxy_info=proxy_info)
+
+            body = (
+                b'User-agent: *\nAllow: /'
+                if is_status_code_client_error(response.status_code)
+                else await response.read()
+            )
+            robots = Protego.parse(body.decode('utf-8'))
+
+        except Exception as e:
+            logger.warning(f'Failed to fetch from robots.txt from "{url}" with error: "{e}"')
 
-        robots = Protego.parse(body.decode('utf-8'))
+            robots = Protego.parse('User-agent: *\nAllow: /')
 
         return cls(url, robots, http_client=http_client, proxy_info=proxy_info)