crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/__init__.py

@@ -1,6 +1,6 @@
 from importlib import metadata
 
-from ._request import Request, RequestOptions
+from ._request import Request, RequestOptions, RequestState
 from ._service_locator import service_locator
 from ._types import ConcurrencySettings, EnqueueStrategy, HttpHeaders, RequestTransformAction, SkippedReason
 from ._utils.globs import Glob
@@ -14,6 +14,7 @@ __all__ = [
     'HttpHeaders',
     'Request',
     'RequestOptions',
+    'RequestState',
     'RequestTransformAction',
     'SkippedReason',
     'service_locator',

crawlee/_autoscaling/snapshotter.py

@@ -113,7 +113,7 @@ class Snapshotter:
         Args:
             config: The `Configuration` instance. Uses the global (default) one if not provided.
         """
-        config = service_locator.get_configuration()
+        config = config or service_locator.get_configuration()
 
         # Compute the maximum memory size based on the provided configuration. If `memory_mbytes` is provided,
         # it uses that value. Otherwise, it calculates the `max_memory_size` as a proportion of the system's

crawlee/_browserforge_workaround.py

@@ -1,4 +1,8 @@
 # ruff: noqa: N802, PLC0415
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 
 def patch_browserforge() -> None:
@@ -12,7 +16,7 @@ def patch_browserforge() -> None:
     import apify_fingerprint_datapoints
     from browserforge import download
 
-    download.DATA_DIRS: dict[str, Path] = {  # type:ignore[misc]
+    download.DATA_DIRS = {
         'headers': apify_fingerprint_datapoints.get_header_network().parent,
         'fingerprints': apify_fingerprint_datapoints.get_fingerprint_network().parent,
     }
@@ -20,7 +24,7 @@ def patch_browserforge() -> None:
     def DownloadIfNotExists(**flags: bool) -> None:
         pass
 
-    download.DownloadIfNotExists = DownloadIfNotExists
+    download.DownloadIfNotExists: Callable[..., None] = DownloadIfNotExists
 
     import browserforge.bayesian_network
 
@@ -33,7 +37,7 @@ def patch_browserforge() -> None:
                 path = download.DATA_DIRS['fingerprints'] / download.DATA_FILES['fingerprints'][path.name]
             super().__init__(path)
 
-    browserforge.bayesian_network.BayesianNetwork = BayesianNetwork  # type:ignore[misc]
+    browserforge.bayesian_network.BayesianNetwork: BayesianNetwork = BayesianNetwork
     import browserforge.headers.generator
 
     browserforge.headers.generator.DATA_DIR = download.DATA_DIRS['headers']

crawlee/_request.py

@@ -11,7 +11,7 @@ from yarl import URL
 from crawlee._types import EnqueueStrategy, HttpHeaders, HttpMethod, HttpPayload, JsonSerializable
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.docs import docs_group
-from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
+from crawlee._utils.requests import compute_unique_key
 from crawlee._utils.urls import validate_http_url
 
 if TYPE_CHECKING:
@@ -34,14 +34,14 @@ class RequestState(IntEnum):
 class CrawleeRequestData(BaseModel):
     """Crawlee-specific configuration stored in the `user_data`."""
 
-    max_retries: Annotated[int | None, Field(alias='maxRetries')] = None
+    max_retries: Annotated[int | None, Field(alias='maxRetries', frozen=True)] = None
     """Maximum number of retries for this request. Allows to override the global `max_request_retries` option of
     `BasicCrawler`."""
 
     enqueue_strategy: Annotated[EnqueueStrategy | None, Field(alias='enqueueStrategy')] = None
     """The strategy that was used for enqueuing the request."""
 
-    state: RequestState | None = None
+    state: RequestState = RequestState.UNPROCESSED
     """Describes the request's current lifecycle state."""
 
     session_rotation_count: Annotated[int | None, Field(alias='sessionRotationCount')] = None
@@ -93,7 +93,7 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]):
     def __delitem__(self, key: str) -> None:
         del self.__pydantic_extra__[key]
 
-    def __iter__(self) -> Iterator[str]:  # type: ignore[override]
+    def __iter__(self) -> Iterator[str]:  # ty: ignore[invalid-method-override]
         yield from self.__pydantic_extra__
 
     def __len__(self) -> int:
@@ -117,6 +117,7 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]):
 user_data_adapter = TypeAdapter(UserData)
 
 
+@docs_group('Other')
 class RequestOptions(TypedDict):
     """Options that can be used to customize request creation.
 
@@ -136,6 +137,8 @@ class RequestOptions(TypedDict):
     always_enqueue: NotRequired[bool]
     user_data: NotRequired[dict[str, JsonSerializable]]
     no_retry: NotRequired[bool]
+    enqueue_strategy: NotRequired[EnqueueStrategy]
+    max_retries: NotRequired[int | None]
 
 
 @docs_group('Storage data')
@@ -163,13 +166,9 @@ class Request(BaseModel):
     ```
     """
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
-    id: str
-    """A unique identifier for the request. Note that this is not used for deduplication, and should not be confused
-    with `unique_key`."""
-
-    unique_key: Annotated[str, Field(alias='uniqueKey')]
+    unique_key: Annotated[str, Field(alias='uniqueKey', frozen=True)]
     """A unique key identifying the request. Two requests with the same `unique_key` are considered as pointing
     to the same URL.
 
@@ -181,40 +180,52 @@ class Request(BaseModel):
     and specify which URLs shall be considered equal.
     """
 
-    url: Annotated[str, BeforeValidator(validate_http_url), Field()]
+    url: Annotated[str, BeforeValidator(validate_http_url), Field(frozen=True)]
     """The URL of the web page to crawl. Must be a valid HTTP or HTTPS URL, and may include query parameters
     and fragments."""
 
-    method: HttpMethod = 'GET'
+    method: Annotated[HttpMethod, Field(frozen=True)] = 'GET'
     """HTTP request method."""
 
-    headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)] = HttpHeaders()
-    """HTTP request headers."""
-
     payload: Annotated[
         HttpPayload | None,
         BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v),
         PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v),
+        Field(frozen=True),
     ] = None
     """HTTP request payload."""
 
-    user_data: Annotated[
-        dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
-        Field(alias='userData', default_factory=lambda: UserData()),
-        PlainValidator(user_data_adapter.validate_python),
-        PlainSerializer(
-            lambda instance: user_data_adapter.dump_python(
-                instance,
-                by_alias=True,
-                exclude_none=True,
-                exclude_unset=True,
-                exclude_defaults=True,
-            )
-        ),
-    ] = {}
-    """Custom user data assigned to the request. Use this to save any request related data to the
-    request's scope, keeping them accessible on retries, failures etc.
-    """
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        headers: HttpHeaders = HttpHeaders()
+        """HTTP request headers."""
+
+        user_data: dict[str, JsonSerializable] = {}
+        """Custom user data assigned to the request. Use this to save any request related data to the
+        request's scope, keeping them accessible on retries, failures etc.
+        """
+
+    else:
+        headers: Annotated[HttpHeaders, Field(default_factory=HttpHeaders)]
+        """HTTP request headers."""
+
+        user_data: Annotated[
+            dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience
+            Field(alias='userData', default_factory=lambda: UserData()),
+            PlainValidator(user_data_adapter.validate_python),
+            PlainSerializer(
+                lambda instance: user_data_adapter.dump_python(
+                    instance,
+                    by_alias=True,
+                    exclude_none=True,
+                    exclude_unset=True,
+                    exclude_defaults=True,
+                )
+            ),
+        ]
+        """Custom user data assigned to the request. Use this to save any request related data to the
+        request's scope, keeping them accessible on retries, failures etc.
+        """
 
     retry_count: Annotated[int, Field(alias='retryCount')] = 0
     """Number of times the request has been retried."""
@@ -239,10 +250,11 @@ class Request(BaseModel):
         label: str | None = None,
         session_id: str | None = None,
         unique_key: str | None = None,
-        id: str | None = None,
         keep_url_fragment: bool = False,
         use_extended_unique_key: bool = False,
         always_enqueue: bool = False,
+        enqueue_strategy: EnqueueStrategy | None = None,
+        max_retries: int | None = None,
         **kwargs: Any,
     ) -> Self:
         """Create a new `Request` instance from a URL.
@@ -264,14 +276,15 @@ class Request(BaseModel):
                 raised.
             unique_key: A unique key identifying the request. If not provided, it is automatically computed based on
                 the URL and other parameters. Requests with the same `unique_key` are treated as identical.
-            id: A unique identifier for the request. If not provided, it is automatically generated from the
-                `unique_key`.
             keep_url_fragment: Determines whether the URL fragment (e.g., `#section`) should be included in
                 the `unique_key` computation. This is only relevant when `unique_key` is not provided.
             use_extended_unique_key: Determines whether to include the HTTP method, ID Session and payload in the
                 `unique_key` computation. This is only relevant when `unique_key` is not provided.
             always_enqueue: If set to `True`, the request will be enqueued even if it is already present in the queue.
                 Using this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`.
+            enqueue_strategy: The strategy that will be used for enqueuing the request.
+            max_retries: Maximum number of retries for this request. Allows to override the global `max_request_retries`
+                option of `BasicCrawler`.
             **kwargs: Additional request properties.
         """
         if unique_key is not None and always_enqueue:
@@ -294,17 +307,29 @@ class Request(BaseModel):
         )
 
         if always_enqueue:
-            unique_key = f'{unique_key}_{crypto_random_object_id()}'
+            unique_key = f'{crypto_random_object_id()}|{unique_key}'
+
+        user_data_dict = kwargs.pop('user_data', {}) or {}
+        crawlee_data_dict = user_data_dict.get('__crawlee', {})
+
+        if max_retries is not None:
+            crawlee_data_dict['maxRetries'] = max_retries
 
-        id = id or unique_key_to_request_id(unique_key)
+        if enqueue_strategy is not None:
+            crawlee_data_dict['enqueueStrategy'] = enqueue_strategy
+
+        crawlee_data = CrawleeRequestData(**crawlee_data_dict)
+
+        if crawlee_data:
+            user_data_dict['__crawlee'] = crawlee_data
 
         request = cls(
             url=url,
             unique_key=unique_key,
-            id=id,
             method=method,
             headers=headers,
             payload=payload,
+            user_data=user_data_dict,
             **kwargs,
         )
 
@@ -350,7 +375,7 @@ class Request(BaseModel):
         self.crawlee_data.crawl_depth = new_value
 
     @property
-    def state(self) -> RequestState | None:
+    def state(self) -> RequestState:
         """Crawlee-specific request handling state."""
         return self.crawlee_data.state
 
@@ -363,10 +388,6 @@ class Request(BaseModel):
         """Crawlee-specific limit on the number of retries of the request."""
         return self.crawlee_data.max_retries
 
-    @max_retries.setter
-    def max_retries(self, new_max_retries: int) -> None:
-        self.crawlee_data.max_retries = new_max_retries
-
     @property
     def session_rotation_count(self) -> int | None:
         """Crawlee-specific number of finished session rotations for the request."""

crawlee/_service_locator.py

@@ -11,6 +11,10 @@ from crawlee.storage_clients import FileSystemStorageClient, StorageClient
 if TYPE_CHECKING:
     from crawlee.storages._storage_instance_manager import StorageInstanceManager
 
+from logging import getLogger
+
+logger = getLogger(__name__)
+
 
 @docs_group('Configuration')
 class ServiceLocator:
@@ -19,23 +23,24 @@ class ServiceLocator:
     All services are initialized to its default value lazily.
     """
 
-    def __init__(self) -> None:
-        self._configuration: Configuration | None = None
-        self._event_manager: EventManager | None = None
-        self._storage_client: StorageClient | None = None
-        self._storage_instance_manager: StorageInstanceManager | None = None
+    global_storage_instance_manager: StorageInstanceManager | None = None
 
-        # Flags to check if the services were already set.
-        self._configuration_was_retrieved = False
-        self._event_manager_was_retrieved = False
-        self._storage_client_was_retrieved = False
+    def __init__(
+        self,
+        configuration: Configuration | None = None,
+        event_manager: EventManager | None = None,
+        storage_client: StorageClient | None = None,
+    ) -> None:
+        self._configuration = configuration
+        self._event_manager = event_manager
+        self._storage_client = storage_client
 
     def get_configuration(self) -> Configuration:
         """Get the configuration."""
         if self._configuration is None:
+            logger.debug('No configuration set, implicitly creating and using default Configuration.')
             self._configuration = Configuration()
 
-        self._configuration_was_retrieved = True
         return self._configuration
 
     def set_configuration(self, configuration: Configuration) -> None:
@@ -47,7 +52,10 @@ class ServiceLocator:
         Raises:
             ServiceConflictError: If the configuration has already been retrieved before.
         """
-        if self._configuration_was_retrieved:
+        if self._configuration is configuration:
+            # Same instance, no need to anything
+            return
+        if self._configuration:
             raise ServiceConflictError(Configuration, configuration, self._configuration)
 
         self._configuration = configuration
@@ -55,13 +63,14 @@ class ServiceLocator:
     def get_event_manager(self) -> EventManager:
         """Get the event manager."""
         if self._event_manager is None:
-            self._event_manager = (
-                LocalEventManager().from_config(config=self._configuration)
-                if self._configuration
-                else LocalEventManager.from_config()
-            )
+            logger.debug('No event manager set, implicitly creating and using default LocalEventManager.')
+            if self._configuration is None:
+                logger.debug(
+                    'Implicit creation of event manager will implicitly set configuration as side effect. '
+                    'It is advised to explicitly first set the configuration instead.'
+                )
+            self._event_manager = LocalEventManager().from_config(config=self._configuration)
 
-        self._event_manager_was_retrieved = True
         return self._event_manager
 
     def set_event_manager(self, event_manager: EventManager) -> None:
@@ -73,7 +82,10 @@ class ServiceLocator:
         Raises:
             ServiceConflictError: If the event manager has already been retrieved before.
         """
-        if self._event_manager_was_retrieved:
+        if self._event_manager is event_manager:
+            # Same instance, no need to anything
+            return
+        if self._event_manager:
             raise ServiceConflictError(EventManager, event_manager, self._event_manager)
 
         self._event_manager = event_manager
@@ -81,9 +93,14 @@ class ServiceLocator:
     def get_storage_client(self) -> StorageClient:
         """Get the storage client."""
         if self._storage_client is None:
+            logger.debug('No storage client set, implicitly creating and using default FileSystemStorageClient.')
+            if self._configuration is None:
+                logger.warning(
+                    'Implicit creation of storage client will implicitly set configuration as side effect. '
+                    'It is advised to explicitly first set the configuration instead.'
+                )
             self._storage_client = FileSystemStorageClient()
 
-        self._storage_client_was_retrieved = True
         return self._storage_client
 
     def set_storage_client(self, storage_client: StorageClient) -> None:
@@ -95,21 +112,24 @@ class ServiceLocator:
         Raises:
             ServiceConflictError: If the storage client has already been retrieved before.
         """
-        if self._storage_client_was_retrieved:
+        if self._storage_client is storage_client:
+            # Same instance, no need to anything
+            return
+        if self._storage_client:
             raise ServiceConflictError(StorageClient, storage_client, self._storage_client)
 
         self._storage_client = storage_client
 
     @property
     def storage_instance_manager(self) -> StorageInstanceManager:
-        """Get the storage instance manager."""
-        if self._storage_instance_manager is None:
+        """Get the storage instance manager. It is global manager shared by all instances of ServiceLocator."""
+        if ServiceLocator.global_storage_instance_manager is None:
             # Import here to avoid circular imports.
             from crawlee.storages._storage_instance_manager import StorageInstanceManager  # noqa: PLC0415
 
-            self._storage_instance_manager = StorageInstanceManager()
+            ServiceLocator.global_storage_instance_manager = StorageInstanceManager()
 
-        return self._storage_instance_manager
+        return ServiceLocator.global_storage_instance_manager
 
 
 service_locator = ServiceLocator()