apify 3.0.0rc1__py3-none-any.whl → 3.0.1b2__py3-none-any.whl

apify/storage_clients/_apify/_request_queue_client.py

@@ -1,62 +1,34 @@
 from __future__ import annotations
 
-import asyncio
-import re
-from base64 import b64encode
-from collections import deque
-from datetime import datetime, timedelta, timezone
-from hashlib import sha256
 from logging import getLogger
-from typing import TYPE_CHECKING, Final
+from typing import TYPE_CHECKING, Final, Literal
 
-from cachetools import LRUCache
 from typing_extensions import override
 
 from apify_client import ApifyClientAsync
+from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
+from crawlee.storages import RequestQueue
 
-from ._models import CachedRequest, ProlongRequestLockResponse, RequestQueueHead
-from apify import Request
+from ._models import ApifyRequestQueueMetadata, RequestQueueStats
+from ._request_queue_shared_client import _ApifyRequestQueueSharedClient
+from ._request_queue_single_client import _ApifyRequestQueueSingleClient
+from ._utils import AliasResolver
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from apify_client.clients import RequestQueueClientAsync
+    from crawlee import Request
 
     from apify import Configuration
 
 logger = getLogger(__name__)
 
 
-def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> str:
-    """Generate a deterministic request ID based on a unique key.
-
-    Args:
-        unique_key: The unique key to convert into a request ID.
-        request_id_length: The length of the request ID.
-
-    Returns:
-        A URL-safe, truncated request ID based on the unique key.
-    """
-    # Encode the unique key and compute its SHA-256 hash
-    hashed_key = sha256(unique_key.encode('utf-8')).digest()
-
-    # Encode the hash in base64 and decode it to get a string
-    base64_encoded = b64encode(hashed_key).decode('utf-8')
-
-    # Remove characters that are not URL-safe ('+', '/', or '=')
-    url_safe_key = re.sub(r'(\+|\/|=)', '', base64_encoded)
-
-    # Truncate the key to the desired length
-    return url_safe_key[:request_id_length]
-
-
 class ApifyRequestQueueClient(RequestQueueClient):
-    """An Apify platform implementation of the request queue client."""
-
-    _DEFAULT_LOCK_TIME: Final[timedelta] = timedelta(minutes=3)
-    """The default lock time for requests in the queue."""
+    """Base class for Apify platform implementations of the request queue client."""
 
     _MAX_CACHED_REQUESTS: Final[int] = 1_000_000
     """Maximum number of requests that can be cached."""
@@ -65,10 +37,8 @@ class ApifyRequestQueueClient(RequestQueueClient):
         self,
         *,
         api_client: RequestQueueClientAsync,
-        id: str,
-        name: str | None,
-        total_request_count: int,
-        handled_request_count: int,
+        metadata: RequestQueueMetadata,
+        access: Literal['single', 'shared'] = 'single',
     ) -> None:
         """Initialize a new instance.
 
@@ -77,179 +47,25 @@ class ApifyRequestQueueClient(RequestQueueClient):
         self._api_client = api_client
         """The Apify request queue client for API operations."""
 
-        self._id = id
-        """The ID of the request queue."""
-
-        self._name = name
-        """The name of the request queue."""
-
-        self._queue_head = deque[str]()
-        """A deque to store request unique keys in the queue head."""
-
-        self._requests_cache: LRUCache[str, CachedRequest] = LRUCache(maxsize=self._MAX_CACHED_REQUESTS)
-        """A cache to store request objects. Request unique key is used as the cache key."""
-
-        self._queue_has_locked_requests: bool | None = None
-        """Whether the queue has requests locked by another client."""
-
-        self._should_check_for_forefront_requests = False
-        """Whether to check for forefront requests in the next list_head call."""
-
-        self._had_multiple_clients = False
-        """Whether the request queue has been accessed by multiple clients."""
-
-        self._initial_total_count = total_request_count
-        """The initial total request count (from the API) when the queue was opened."""
-
-        self._initial_handled_count = handled_request_count
-        """The initial handled request count (from the API) when the queue was opened."""
-
-        self._assumed_total_count = 0
-        """The number of requests we assume are in the queue (tracked manually for this instance)."""
-
-        self._assumed_handled_count = 0
-        """The number of requests we assume have been handled (tracked manually for this instance)."""
-
-        self._fetch_lock = asyncio.Lock()
-        """Fetch lock to minimize race conditions when communicating with API."""
-
-    @override
-    async def get_metadata(self) -> RequestQueueMetadata:
-        total_count = self._initial_total_count + self._assumed_total_count
-        handled_count = self._initial_handled_count + self._assumed_handled_count
-        pending_count = total_count - handled_count
-
-        return RequestQueueMetadata(
-            id=self._id,
-            name=self._name,
-            total_request_count=total_count,
-            handled_request_count=handled_count,
-            pending_request_count=pending_count,
-            created_at=datetime.now(timezone.utc),
-            modified_at=datetime.now(timezone.utc),
-            accessed_at=datetime.now(timezone.utc),
-            had_multiple_clients=self._had_multiple_clients,
-        )
-
-    @classmethod
-    async def open(
-        cls,
-        *,
-        id: str | None,
-        name: str | None,
-        configuration: Configuration,
-    ) -> ApifyRequestQueueClient:
-        """Open an Apify request queue client.
-
-        This method creates and initializes a new instance of the Apify request queue client. It handles
-        authentication, storage lookup/creation, and metadata retrieval, and sets up internal caching and queue
-        management structures.
-
-        Args:
-            id: The ID of an existing request queue to open. If provided, the client will connect to this specific
-                storage. Cannot be used together with `name`.
-            name: The name of a request queue to get or create. If a storage with this name exists, it will be opened;
-                otherwise, a new one will be created. Cannot be used together with `id`.
-            configuration: The configuration object containing API credentials and settings. Must include a valid
-                `token` and `api_base_url`. May also contain a `default_request_queue_id` for fallback when neither
-                `id` nor `name` is provided.
-
-        Returns:
-            An instance for the opened or created storage client.
-
-        Raises:
-            ValueError: If the configuration is missing required fields (token, api_base_url), if both `id` and `name`
-                are provided, or if neither `id` nor `name` is provided and no default storage ID is available
-                in the configuration.
-        """
-        token = configuration.token
-        if not token:
-            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
-
-        api_url = configuration.api_base_url
-        if not api_url:
-            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
-
-        api_public_base_url = configuration.api_public_base_url
-        if not api_public_base_url:
-            raise ValueError(
-                'Apify storage client requires a valid API public base URL in Configuration '
-                f'(api_public_base_url={api_public_base_url}).'
+        self._implementation: _ApifyRequestQueueSingleClient | _ApifyRequestQueueSharedClient
+        """Internal implementation used to communicate with the Apify platform based Request Queue."""
+        if access == 'single':
+            self._implementation = _ApifyRequestQueueSingleClient(
+                api_client=self._api_client, metadata=metadata, cache_size=self._MAX_CACHED_REQUESTS
             )
+        elif access == 'shared':
+            self._implementation = _ApifyRequestQueueSharedClient(
+                api_client=self._api_client,
+                metadata=metadata,
+                cache_size=self._MAX_CACHED_REQUESTS,
+                metadata_getter=self.get_metadata,
+            )
+        else:
+            raise RuntimeError(f"Unsupported access type: {access}. Allowed values are 'single' or 'shared'.")
 
-        # Create Apify client with the provided token and API URL.
-        apify_client_async = ApifyClientAsync(
-            token=token,
-            api_url=api_url,
-            max_retries=8,
-            min_delay_between_retries_millis=500,
-            timeout_secs=360,
-        )
-        apify_rqs_client = apify_client_async.request_queues()
-
-        # If both id and name are provided, raise an error.
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
-        # If id is provided, get the storage by ID.
-        if id and name is None:
-            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
-
-        # If name is provided, get or create the storage by name.
-        if name and id is None:
-            id = RequestQueueMetadata.model_validate(
-                await apify_rqs_client.get_or_create(name=name),
-            ).id
-            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
-
-        # If both id and name are None, try to get the default storage ID from environment variables.
-        # The default storage ID environment variable is set by the Apify platform. It also contains
-        # a new storage ID after Actor's reboot or migration.
-        if id is None and name is None:
-            id = configuration.default_request_queue_id
-            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
-
-        # Fetch its metadata.
-        metadata = await apify_rq_client.get()
-
-        # If metadata is None, it means the storage does not exist, so we create it.
-        if metadata is None:
-            id = RequestQueueMetadata.model_validate(
-                await apify_rqs_client.get_or_create(),
-            ).id
-            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
-
-        # Verify that the storage exists by fetching its metadata again.
-        metadata = await apify_rq_client.get()
-        if metadata is None:
-            raise ValueError(f'Opening request queue with id={id} and name={name} failed.')
-
-        metadata_model = RequestQueueMetadata.model_validate(
-            await apify_rqs_client.get_or_create(),
-        )
-
-        # Ensure we have a valid ID.
-        if id is None:
-            raise ValueError('Request queue ID cannot be None.')
-
-        return cls(
-            api_client=apify_rq_client,
-            id=id,
-            name=name,
-            total_request_count=metadata_model.total_request_count,
-            handled_request_count=metadata_model.handled_request_count,
-        )
-
-    @override
-    async def purge(self) -> None:
-        raise NotImplementedError(
-            'Purging the request queue is not supported in the Apify platform. '
-            'Use the `drop` method to delete the request queue instead.'
-        )
-
-    @override
-    async def drop(self) -> None:
-        await self._api_client.delete()
+    @property
+    def _metadata(self) -> RequestQueueMetadata:
+        return self._implementation.metadata
 
     @override
     async def add_batch_of_requests(
@@ -267,100 +83,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
         Returns:
             Response containing information about the added requests.
         """
-        # Do not try to add previously added requests to avoid pointless expensive calls to API
-
-        new_requests: list[Request] = []
-        already_present_requests: list[ProcessedRequest] = []
-
-        for request in requests:
-            if self._requests_cache.get(request.unique_key):
-                # We are not sure if it was already handled at this point, and it is not worth calling API for it.
-                # It could have been handled by another client in the meantime, so cached information about
-                # `request.was_already_handled` is not reliable.
-                already_present_requests.append(
-                    ProcessedRequest.model_validate(
-                        {
-                            'uniqueKey': request.unique_key,
-                            'wasAlreadyPresent': True,
-                            'wasAlreadyHandled': request.was_already_handled,
-                        }
-                    )
-                )
-
-            else:
-                # Add new request to the cache.
-                processed_request = ProcessedRequest.model_validate(
-                    {
-                        'uniqueKey': request.unique_key,
-                        'wasAlreadyPresent': True,
-                        'wasAlreadyHandled': request.was_already_handled,
-                    }
-                )
-                self._cache_request(
-                    request.unique_key,
-                    processed_request,
-                )
-                new_requests.append(request)
-
-        if new_requests:
-            # Prepare requests for API by converting to dictionaries.
-            requests_dict = [
-                request.model_dump(
-                    by_alias=True,
-                    exclude={'id'},  # Exclude ID fields from requests since the API doesn't accept them.
-                )
-                for request in new_requests
-            ]
-
-            # Send requests to API.
-            api_response = AddRequestsResponse.model_validate(
-                await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
-            )
-
-            # Add the locally known already present processed requests based on the local cache.
-            api_response.processed_requests.extend(already_present_requests)
-
-            # Remove unprocessed requests from the cache
-            for unprocessed_request in api_response.unprocessed_requests:
-                self._requests_cache.pop(unprocessed_request.unique_key, None)
-
-        else:
-            api_response = AddRequestsResponse.model_validate(
-                {'unprocessedRequests': [], 'processedRequests': already_present_requests}
-            )
-
-        logger.debug(
-            f'Tried to add new requests: {len(new_requests)}, '
-            f'succeeded to add new requests: {len(api_response.processed_requests) - len(already_present_requests)}, '
-            f'skipped already present requests: {len(already_present_requests)}'
-        )
-
-        # Update assumed total count for newly added requests.
-        new_request_count = 0
-        for processed_request in api_response.processed_requests:
-            if not processed_request.was_already_present and not processed_request.was_already_handled:
-                new_request_count += 1
-
-        self._assumed_total_count += new_request_count
-
-        return api_response
-
-    @override
-    async def get_request(self, unique_key: str) -> Request | None:
-        """Get a request by unique key.
-
-        Args:
-            unique_key: Unique key of the request to get.
-
-        Returns:
-            The request or None if not found.
-        """
-        response = await self._api_client.get_request(unique_key_to_request_id(unique_key))
-
-        if response is None:
-            return None
-
-        return Request.model_validate(response)
+        return await self._implementation.add_batch_of_requests(requests, forefront=forefront)
 
     @override
     async def fetch_next_request(self) -> Request | None:
@@ -374,45 +97,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
         Returns:
             The request or `None` if there are no more pending requests.
         """
-        # Ensure the queue head has requests if available. Fetching the head with lock to prevent race conditions.
-        async with self._fetch_lock:
-            await self._ensure_head_is_non_empty()
-
-            # If queue head is empty after ensuring, there are no requests
-            if not self._queue_head:
-                return None
-
-            # Get the next request ID from the queue head
-            next_unique_key = self._queue_head.popleft()
-
-        request = await self._get_or_hydrate_request(next_unique_key)
-
-        # Handle potential inconsistency where request might not be in the main table yet
-        if request is None:
-            logger.debug(
-                'Cannot find a request from the beginning of queue, will be retried later',
-                extra={'nextRequestUniqueKey': next_unique_key},
-            )
-            return None
-
-        # If the request was already handled, skip it
-        if request.handled_at is not None:
-            logger.debug(
-                'Request fetched from the beginning of queue was already handled',
-                extra={'nextRequestUniqueKey': next_unique_key},
-            )
-            return None
-
-        # Use get request to ensure we have the full request object.
-        request = await self.get_request(request.unique_key)
-        if request is None:
-            logger.debug(
-                'Request fetched from the beginning of queue was not found in the RQ',
-                extra={'nextRequestUniqueKey': next_unique_key},
-            )
-            return None
-
-        return request
+        return await self._implementation.fetch_next_request()
 
     @override
     async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None:
@@ -426,33 +111,19 @@ class ApifyRequestQueueClient(RequestQueueClient):
         Returns:
             Information about the queue operation. `None` if the given request was not in progress.
         """
-        # Set the handled_at timestamp if not already set
-        if request.handled_at is None:
-            request.handled_at = datetime.now(tz=timezone.utc)
-
-        if cached_request := self._requests_cache[request.unique_key]:
-            cached_request.was_already_handled = request.was_already_handled
-        try:
-            # Update the request in the API
-            processed_request = await self._update_request(request)
-            processed_request.unique_key = request.unique_key
-
-            # Update assumed handled count if this wasn't already handled
-            if not processed_request.was_already_handled:
-                self._assumed_handled_count += 1
-
-            # Update the cache with the handled request
-            cache_key = request.unique_key
-            self._cache_request(
-                cache_key,
-                processed_request,
-                hydrated_request=request,
-            )
-        except Exception as exc:
-            logger.debug(f'Error marking request {request.unique_key} as handled: {exc!s}')
-            return None
-        else:
-            return processed_request
+        return await self._implementation.mark_request_as_handled(request)
+
+    @override
+    async def get_request(self, unique_key: str) -> Request | None:
+        """Get a request by unique key.
+
+        Args:
+            unique_key: Unique key of the request to get.
+
+        Returns:
+            The request or None if not found.
+        """
+        return await self._implementation.get_request(unique_key)
 
     @override
     async def reclaim_request(
@@ -472,46 +143,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
         Returns:
             Information about the queue operation. `None` if the given request was not in progress.
         """
-        # Check if the request was marked as handled and clear it. When reclaiming,
-        # we want to put the request back for processing.
-        if request.was_already_handled:
-            request.handled_at = None
-
-        # Reclaim with lock to prevent race conditions that could lead to double processing of the same request.
-        async with self._fetch_lock:
-            try:
-                # Update the request in the API.
-                processed_request = await self._update_request(request, forefront=forefront)
-                processed_request.unique_key = request.unique_key
-
-                # If the request was previously handled, decrement our handled count since
-                # we're putting it back for processing.
-                if request.was_already_handled and not processed_request.was_already_handled:
-                    self._assumed_handled_count -= 1
-
-                # Update the cache
-                cache_key = request.unique_key
-                self._cache_request(
-                    cache_key,
-                    processed_request,
-                    hydrated_request=request,
-                )
-
-                # If we're adding to the forefront, we need to check for forefront requests
-                # in the next list_head call
-                if forefront:
-                    self._should_check_for_forefront_requests = True
-
-                # Try to release the lock on the request
-                try:
-                    await self._delete_request_lock(request.unique_key, forefront=forefront)
-                except Exception as err:
-                    logger.debug(f'Failed to delete request lock for request {request.unique_key}', exc_info=err)
-            except Exception as exc:
-                logger.debug(f'Error reclaiming request {request.unique_key}: {exc!s}')
-                return None
-            else:
-                return processed_request
+        return await self._implementation.reclaim_request(request, forefront=forefront)
 
     @override
     async def is_empty(self) -> bool:
@@ -520,268 +152,176 @@ class ApifyRequestQueueClient(RequestQueueClient):
         Returns:
             True if the queue is empty, False otherwise.
         """
-        # Check _list_head and self._queue_has_locked_requests with lock to make sure they are consistent.
-        # Without the lock the `is_empty` is prone to falsely report True with some low probability race condition.
-        async with self._fetch_lock:
-            head = await self._list_head(limit=1, lock_time=None)
-            return len(head.items) == 0 and not self._queue_has_locked_requests
-
-    async def _ensure_head_is_non_empty(self) -> None:
-        """Ensure that the queue head has requests if they are available in the queue."""
-        # If queue head has adequate requests, skip fetching more
-        if len(self._queue_head) > 1 and not self._should_check_for_forefront_requests:
-            return
-
-        # Fetch requests from the API and populate the queue head
-        await self._list_head(lock_time=self._DEFAULT_LOCK_TIME)
-
-    async def _get_or_hydrate_request(self, unique_key: str) -> Request | None:
-        """Get a request by unique key, either from cache or by fetching from API.
-
-        Args:
-            unique_key: Unique keu of the request to get.
-
-        Returns:
-            The request if found and valid, otherwise None.
-        """
-        # First check if the request is in our cache
-        cached_entry = self._requests_cache.get(unique_key)
-
-        if cached_entry and cached_entry.hydrated:
-            # If we have the request hydrated in cache, check if lock is expired
-            if cached_entry.lock_expires_at and cached_entry.lock_expires_at < datetime.now(tz=timezone.utc):
-                # Try to prolong the lock if it's expired
-                try:
-                    lock_secs = int(self._DEFAULT_LOCK_TIME.total_seconds())
-                    response = await self._prolong_request_lock(unique_key, lock_secs=lock_secs)
-                    cached_entry.lock_expires_at = response.lock_expires_at
-                except Exception:
-                    # If prolonging the lock fails, we lost the request
-                    logger.debug(f'Failed to prolong lock for request {unique_key}, returning None')
-                    return None
-
-            return cached_entry.hydrated
-
-        # If not in cache or not hydrated, fetch the request
-        try:
-            # Try to acquire or prolong the lock
-            lock_secs = int(self._DEFAULT_LOCK_TIME.total_seconds())
-            await self._prolong_request_lock(unique_key, lock_secs=lock_secs)
-
-            # Fetch the request data
-            request = await self.get_request(unique_key)
-
-            # If request is not found, release lock and return None
-            if not request:
-                await self._delete_request_lock(unique_key)
-                return None
-
-            # Update cache with hydrated request
-            cache_key = request.unique_key
-            self._cache_request(
-                cache_key,
-                ProcessedRequest(
-                    unique_key=request.unique_key,
-                    was_already_present=True,
-                    was_already_handled=request.handled_at is not None,
-                ),
-                hydrated_request=request,
-            )
-        except Exception as exc:
-            logger.debug(f'Error fetching or locking request {unique_key}: {exc!s}')
-            return None
-        else:
-            return request
+        return await self._implementation.is_empty()
 
-    async def _update_request(
-        self,
-        request: Request,
-        *,
-        forefront: bool = False,
-    ) -> ProcessedRequest:
-        """Update a request in the queue.
-
-        Args:
-            request: The updated request.
-            forefront: Whether to put the updated request in the beginning or the end of the queue.
+    @override
+    async def get_metadata(self) -> ApifyRequestQueueMetadata:
+        """Get metadata about the request queue.
 
         Returns:
-            The updated request
+            Metadata from the API, merged with local estimation, because in some cases, the data from the API can
+            be delayed.
         """
-        request_dict = request.model_dump(by_alias=True)
-        request_dict['id'] = unique_key_to_request_id(request.unique_key)
-        response = await self._api_client.update_request(
-            request=request_dict,
-            forefront=forefront,
-        )
-
-        return ProcessedRequest.model_validate(
-            {'uniqueKey': request.unique_key} | response,
+        response = await self._api_client.get()
+        if response is None:
+            raise ValueError('Failed to fetch request queue metadata from the API.')
+        # Enhance API response by local estimations (API can be delayed few seconds, while local estimation not.)
+        return ApifyRequestQueueMetadata(
+            id=response['id'],
+            name=response['name'],
+            total_request_count=max(response['totalRequestCount'], self._metadata.total_request_count),
+            handled_request_count=max(response['handledRequestCount'], self._metadata.handled_request_count),
+            pending_request_count=response['pendingRequestCount'],
+            created_at=min(response['createdAt'], self._metadata.created_at),
+            modified_at=max(response['modifiedAt'], self._metadata.modified_at),
+            accessed_at=max(response['accessedAt'], self._metadata.accessed_at),
+            had_multiple_clients=response['hadMultipleClients'] or self._metadata.had_multiple_clients,
+            stats=RequestQueueStats.model_validate(response['stats'], by_alias=True),
         )
 
-    async def _list_head(
-        self,
+    @classmethod
+    async def open(
+        cls,
         *,
-        lock_time: timedelta | None = None,
-        limit: int = 25,
-    ) -> RequestQueueHead:
-        """Retrieve requests from the beginning of the queue.
+        id: str | None,
+        name: str | None,
+        alias: str | None,
+        configuration: Configuration,
+        access: Literal['single', 'shared'] = 'single',
+    ) -> ApifyRequestQueueClient:
+        """Open an Apify request queue client.
+
+        This method creates and initializes a new instance of the Apify request queue client. It handles
+        authentication, storage lookup/creation, and metadata retrieval, and sets up internal caching and queue
+        management structures.
 
         Args:
-            lock_time: Duration for which to lock the retrieved requests.
-                If None, requests will not be locked.
-            limit: Maximum number of requests to retrieve.
+            id: The ID of the RQ to open. If provided, searches for existing RQ by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the RQ to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the RQ to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
+            configuration: The configuration object containing API credentials and settings. Must include a valid
+                `token` and `api_base_url`. May also contain a `default_request_queue_id` for fallback when neither
+                `id`, `name`, nor `alias` is provided.
+            access: Controls the implementation of the request queue client based on expected scenario:
+                - 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
+                - 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
+                Detailed constraints for the 'single' access type:
+                - Only one client is consuming the request queue at the time.
+                - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to
+                  be handled so quickly as this client does not aggressively fetch the forefront and relies on local
+                  head estimation.
+                - Requests are only added to the queue, never deleted by other clients. (Marking as handled is ok.)
+                - Other producers can add new requests, but not modify existing ones.
+                  (Modifications would not be included in local cache)
 
         Returns:
-            A collection of requests from the beginning of the queue.
+            An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
+                `id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
+                in the configuration.
         """
-        # Return from cache if available and we're not checking for new forefront requests
-        if self._queue_head and not self._should_check_for_forefront_requests:
-            logger.debug(f'Using cached queue head with {len(self._queue_head)} requests')
-            # Create a list of requests from the cached queue head
-            items = []
-            for unique_key in list(self._queue_head)[:limit]:
-                cached_request = self._requests_cache.get(unique_key)
-                if cached_request and cached_request.hydrated:
-                    items.append(cached_request.hydrated)
-
-            metadata = await self.get_metadata()
-
-            return RequestQueueHead(
-                limit=limit,
-                had_multiple_clients=metadata.had_multiple_clients,
-                queue_modified_at=metadata.modified_at,
-                items=items,
-                queue_has_locked_requests=self._queue_has_locked_requests,
-                lock_time=lock_time,
-            )
-        leftover_buffer = list[str]()
-        if self._should_check_for_forefront_requests:
-            leftover_buffer = list(self._queue_head)
-            self._queue_head.clear()
-            self._should_check_for_forefront_requests = False
-
-        # Otherwise fetch from API
-        lock_time = lock_time or self._DEFAULT_LOCK_TIME
-        lock_secs = int(lock_time.total_seconds())
-
-        response = await self._api_client.list_and_lock_head(
-            lock_secs=lock_secs,
-            limit=limit,
-        )
+        if sum(1 for param in [id, name, alias] if param is not None) > 1:
+            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
 
-        # Update the queue head cache
-        self._queue_has_locked_requests = response.get('queueHasLockedRequests', False)
+        token = configuration.token
+        if not token:
+            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
 
-        for request_data in response.get('items', []):
-            request = Request.model_validate(request_data)
+        api_url = configuration.api_base_url
+        if not api_url:
+            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
 
-            # Skip requests without ID or unique key
-            if not request.unique_key:
-                logger.debug(
-                    'Skipping request from queue head, missing ID or unique key',
-                    extra={
-                        'unique_key': request.unique_key,
-                    },
-                )
-                continue
-
-            # Cache the request
-            self._cache_request(
-                request.unique_key,
-                ProcessedRequest(
-                    unique_key=request.unique_key,
-                    was_already_present=True,
-                    was_already_handled=False,
-                ),
-                hydrated_request=request,
+        api_public_base_url = configuration.api_public_base_url
+        if not api_public_base_url:
+            raise ValueError(
+                'Apify storage client requires a valid API public base URL in Configuration '
+                f'(api_public_base_url={api_public_base_url}).'
             )
-            self._queue_head.append(request.unique_key)
 
-        for leftover_unique_key in leftover_buffer:
-            # After adding new requests to the forefront, any existing leftover locked request is kept in the end.
-            self._queue_head.append(leftover_unique_key)
-        return RequestQueueHead.model_validate(response)
+        # Create Apify client with the provided token and API URL.
+        apify_client_async = ApifyClientAsync(
+            token=token,
+            api_url=api_url,
+            max_retries=8,
+            min_delay_between_retries_millis=500,
+            timeout_secs=360,
+        )
+        apify_rqs_client = apify_client_async.request_queues()
 
-    async def _prolong_request_lock(
-        self,
-        unique_key: str,
-        *,
-        lock_secs: int,
-    ) -> ProlongRequestLockResponse:
-        """Prolong the lock on a specific request in the queue.
+        # Normalize unnamed default storage in cases where not defined in `configuration.default_request_queue_id` to
+        # unnamed storage aliased as `__default__`
+        if not any([alias, name, id, configuration.default_request_queue_id]):
+            alias = '__default__'
+
+        if alias:
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=RequestQueue, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = RequestQueueMetadata.model_validate(
+                        await apify_rqs_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
-        Args:
-            unique_key: Unique key of the request whose lock is to be prolonged.
-            lock_secs: The additional amount of time, in seconds, that the request will remain locked.
+        # If name is provided, get or create the storage by name.
+        elif name:
+            id = RequestQueueMetadata.model_validate(
+                await apify_rqs_client.get_or_create(name=name),
+            ).id
 
-        Returns:
-            A response containing the time at which the lock will expire.
-        """
-        response = await self._api_client.prolong_request_lock(
-            request_id=unique_key_to_request_id(unique_key),
-            # All requests reaching this code were the tip of the queue at the moment when they were fetched,
-            # so if their lock expires, they should be put back to the forefront as their handling is long overdue.
-            forefront=True,
-            lock_secs=lock_secs,
-        )
+        # If none are provided, try to get the default storage ID from environment variables.
+        elif id is None:
+            id = configuration.default_request_queue_id
+            if not id:
+                raise ValueError(
+                    'RequestQueue "id", "name", or "alias" must be specified, '
+                    'or a default default_request_queue_id ID must be set in the configuration.'
+                )
 
-        result = ProlongRequestLockResponse(
-            lock_expires_at=datetime.fromisoformat(response['lockExpiresAt'].replace('Z', '+00:00'))
-        )
+        # Use suitable client_key to make `hadMultipleClients` response of Apify API useful.
+        # It should persist across migrated or resurrected Actor runs on the Apify platform.
+        _api_max_client_key_length = 32
+        client_key = (configuration.actor_run_id or crypto_random_object_id(length=_api_max_client_key_length))[
+            :_api_max_client_key_length
+        ]
 
-        # Update the cache with the new lock expiration
-        for cached_request in self._requests_cache.values():
-            if cached_request.unique_key == unique_key:
-                cached_request.lock_expires_at = result.lock_expires_at
-                break
+        apify_rq_client = apify_client_async.request_queue(request_queue_id=id, client_key=client_key)
 
-        return result
+        # Fetch its metadata.
+        metadata = await apify_rq_client.get()
 
-    async def _delete_request_lock(
-        self,
-        unique_key: str,
-        *,
-        forefront: bool = False,
-    ) -> None:
-        """Delete the lock on a specific request in the queue.
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = RequestQueueMetadata.model_validate(
+                await apify_rqs_client.get_or_create(),
+            ).id
+            apify_rq_client = apify_client_async.request_queue(request_queue_id=id, client_key=client_key)
 
-        Args:
-            unique_key: Unique key of the request to delete the lock.
-            forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted.
-        """
-        try:
-            await self._api_client.delete_request_lock(
-                request_id=unique_key_to_request_id(unique_key),
-                forefront=forefront,
-            )
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_rq_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening request queue with id={id}, name={name}, and alias={alias} failed.')
 
-            # Update the cache to remove the lock
-            for cached_request in self._requests_cache.values():
-                if cached_request.unique_key == unique_key:
-                    cached_request.lock_expires_at = None
-                    break
-        except Exception as err:
-            logger.debug(f'Failed to delete request lock for request {unique_key}', exc_info=err)
+        metadata_model = RequestQueueMetadata.model_validate(metadata)
 
-    def _cache_request(
-        self,
-        cache_key: str,
-        processed_request: ProcessedRequest,
-        *,
-        hydrated_request: Request | None = None,
-    ) -> None:
-        """Cache a request for future use.
+        return cls(api_client=apify_rq_client, metadata=metadata_model, access=access)
 
-        Args:
-            cache_key: The key to use for caching the request. It should be request ID.
-            processed_request: The processed request information.
-            forefront: Whether the request was added to the forefront of the queue.
-            hydrated_request: The hydrated request object, if available.
-        """
-        self._requests_cache[cache_key] = CachedRequest(
-            unique_key=processed_request.unique_key,
-            was_already_handled=processed_request.was_already_handled,
-            hydrated=hydrated_request,
-            lock_expires_at=None,
+    @override
+    async def purge(self) -> None:
+        raise NotImplementedError(
+            'Purging the request queue is not supported in the Apify platform. '
+            'Use the `drop` method to delete the request queue instead.'
         )
+
+    @override
+    async def drop(self) -> None:
+        await self._api_client.delete()