apify 3.0.1b1__py3-none-any.whl → 3.0.2__py3-none-any.whl

apify/_actor.py

@@ -32,7 +32,7 @@ from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
 from apify._models import ActorRun
 from apify._proxy_configuration import ProxyConfiguration
-from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython, maybe_extract_enum_member_value
+from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
 from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
 from apify.storage_clients import ApifyStorageClient
@@ -48,10 +48,10 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
+    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
-
 MainReturnType = TypeVar('MainReturnType')
 
 
@@ -98,7 +98,10 @@ class _ActorType:
     """
 
     _is_rebooting = False
+    """Whether the Actor is currently rebooting."""
+
     _is_any_instance_initialized = False
+    """Whether any Actor instance was initialized."""
 
     def __init__(
         self,
@@ -106,63 +109,149 @@ class _ActorType:
         *,
         configure_logging: bool = True,
         exit_process: bool | None = None,
+        exit_code: int = 0,
+        status_message: str | None = None,
+        event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT,
+        cleanup_timeout: timedelta = timedelta(seconds=30),
     ) -> None:
-        """Create an Actor instance.
-
-        Note that you don't have to do this, all the functionality is accessible using the default instance
-        (e.g. `Actor.open_dataset()`).
+        """Initialize a new instance.
 
         Args:
-            configuration: The Actor configuration to be used. If not passed, a new Configuration instance will
-                be created.
-            configure_logging: Should the default logging configuration be configured?
-            exit_process: Whether the Actor should call `sys.exit` when the context manager exits. The default is
-                True except for the IPython, Pytest and Scrapy environments.
+            configuration: The Actor configuration to use. If not provided, a default configuration is created.
+            configure_logging: Whether to set up the default logging configuration.
+            exit_process: Whether the Actor should call `sys.exit` when the context manager exits.
+                Defaults to True, except in IPython, Pytest, and Scrapy environments.
+            exit_code: The exit code the Actor should use when exiting.
+            status_message: Final status message to display upon Actor termination.
+            event_listeners_timeout: Maximum time to wait for Actor event listeners to complete before exiting.
+            cleanup_timeout: Maximum time to wait for cleanup tasks to finish.
         """
+        self._configuration = configuration
+        self._configure_logging = configure_logging
         self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process
-        self._is_exiting = False
+        self._exit_code = exit_code
+        self._status_message = status_message
+        self._event_listeners_timeout = event_listeners_timeout
+        self._cleanup_timeout = cleanup_timeout
 
         # Actor state when this method is being executed is unpredictable.
         # Actor can be initialized by lazy object proxy or by user directly, or by both.
         # Until `init` method is run, this state of uncertainty remains. This is the reason why any setting done here in
         # `__init__` method should not be considered final.
 
-        self._configuration = configuration
-        self._configure_logging = configure_logging
         self._apify_client: ApifyClientAsync | None = None
+        self._local_storage_client: StorageClient | None = None
 
+        self._is_exiting = False
         self._is_initialized = False
 
     async def __aenter__(self) -> Self:
-        """Initialize the Actor.
+        """Enter the Actor context.
+
+        Initializes the Actor when used in an `async with` block. This method:
 
-        Automatically initializes the Actor instance when you use it in an `async with ...` statement.
+        - Sets up local or cloud storage clients depending on whether the Actor runs locally or on the Apify platform.
+        - Configures the event manager and starts periodic state persistence.
+        - Initializes the charging manager for handling charging events.
+        - Configures logging after all core services are registered.
 
-        When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while
-        executing the block code, the `Actor.fail` method is called.
+        This method must be called exactly once per Actor instance. Re-initializing an Actor or having multiple
+        active Actor instances is not standard usage and may lead to warnings or unexpected behavior.
         """
-        await self.init()
+        if self._is_initialized:
+            raise RuntimeError('The Actor was already initialized!')
+
+        # Initialize configuration first - it's required for the next steps.
+        if self._configuration:
+            # User provided explicit configuration - register it in the service locator.
+            service_locator.set_configuration(self.configuration)
+        else:
+            # No explicit configuration provided - trigger creation of default configuration.
+            _ = self.configuration
+
+        # Configure logging based on the configuration, any logs before this point are lost.
+        if self._configure_logging:
+            _configure_logging()
+        self.log.debug('Logging configured')
+
+        self.log.info('Initializing Actor', extra=get_system_info())
+        self.log.debug('Configuration initialized')
+
+        # Warn about non-standard usage patterns.
+        if _ActorType._is_any_instance_initialized:
+            self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care.')
+
+        # Update the global Actor proxy to refer to this instance.
+        cast('Proxy', Actor).__wrapped__ = self
+        self._is_exiting = False
+        self._was_final_persist_state_emitted = False
+
+        # Initialize the storage client and register it in the service locator.
+        _ = self._storage_client
+        self.log.debug('Storage client initialized')
+
+        # Initialize the event manager and register it in the service locator.
+        await self.event_manager.__aenter__()
+        self.log.debug('Event manager initialized')
+
+        # Initialize the charging manager.
+        await self._charging_manager_implementation.__aenter__()
+        self.log.debug('Charging manager initialized')
+
+        # Mark initialization as complete and update global state.
+        self._is_initialized = True
+        _ActorType._is_any_instance_initialized = True
         return self
 
     async def __aexit__(
         self,
-        _exc_type: type[BaseException] | None,
+        exc_type: type[BaseException] | None,
         exc_value: BaseException | None,
-        _exc_traceback: TracebackType | None,
+        exc_traceback: TracebackType | None,
     ) -> None:
-        """Exit the Actor, handling any exceptions properly.
+        """Exit the Actor context.
+
+        If the block exits with an exception, the Actor fails with a non-zero exit code.
+        Otherwise, it exits cleanly. In both cases the Actor:
 
-        When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while
-        executing the block code, the `Actor.fail` method is called.
+        - Cancels periodic `PERSIST_STATE` events.
+        - Sends a final `PERSIST_STATE` event.
+        - Waits for all event listeners to finish.
+        - Stops the event manager and the charging manager.
+        - Optionally terminates the process with the selected exit code.
         """
-        if not self._is_exiting:
-            if exc_value:
-                await self.fail(
-                    exit_code=ActorExitCodes.ERROR_USER_FUNCTION_THREW.value,
-                    exception=exc_value,
-                )
-            else:
-                await self.exit()
+        if self._is_exiting:
+            return
+
+        self._raise_if_not_initialized()
+
+        if exc_value and not is_running_in_ipython():
+            # In IPython, we don't run `sys.exit()` during Actor exits,
+            # so the exception traceback will be printed on its own
+            self.log.exception('Actor failed with an exception', exc_info=exc_value)
+            self.exit_code = ActorExitCodes.ERROR_USER_FUNCTION_THREW.value
+
+        self._is_exiting = True
+        self.log.info('Exiting Actor', extra={'exit_code': self.exit_code})
+
+        async def finalize() -> None:
+            if self.status_message is not None:
+                await self.set_status_message(self.status_message, is_terminal=True)
+
+            # Sleep for a bit so that the listeners have a chance to trigger
+            await asyncio.sleep(0.1)
+
+            if self._event_listeners_timeout:
+                await self.event_manager.wait_for_all_listeners_to_complete(timeout=self._event_listeners_timeout)
+
+            await self.event_manager.__aexit__(None, None, None)
+            await self._charging_manager_implementation.__aexit__(None, None, None)
+
+        await asyncio.wait_for(finalize(), self._cleanup_timeout.total_seconds())
+        self._is_initialized = False
+
+        if self._exit_process:
+            sys.exit(self.exit_code)
 
     def __repr__(self) -> str:
         if self is cast('Proxy', Actor).__wrapped__:
@@ -176,24 +265,58 @@ class _ActorType:
         *,
         configure_logging: bool = True,
         exit_process: bool | None = None,
+        exit_code: int = 0,
+        event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT,
+        status_message: str | None = None,
+        cleanup_timeout: timedelta = timedelta(seconds=30),
     ) -> Self:
-        """Make a new Actor instance with a non-default configuration."""
+        """Make a new Actor instance with a non-default configuration.
+
+        This is necessary due to the lazy object proxying of the global `Actor` instance.
+        """
         return self.__class__(
             configuration=configuration,
             configure_logging=configure_logging,
             exit_process=exit_process,
+            exit_code=exit_code,
+            event_listeners_timeout=event_listeners_timeout,
+            status_message=status_message,
+            cleanup_timeout=cleanup_timeout,
         )
 
+    @property
+    def log(self) -> logging.Logger:
+        """Logger configured for this Actor."""
+        return logger
+
+    @property
+    def exit_code(self) -> int:
+        """The exit code the Actor will use when exiting."""
+        return self._exit_code
+
+    @exit_code.setter
+    def exit_code(self, value: int) -> None:
+        self._exit_code = value
+
+    @property
+    def status_message(self) -> str | None:
+        """The final status message that the Actor will display upon termination."""
+        return self._status_message
+
+    @status_message.setter
+    def status_message(self, value: str | None) -> None:
+        self._status_message = value
+
     @property
     def apify_client(self) -> ApifyClientAsync:
-        """The ApifyClientAsync instance the Actor instance uses."""
+        """Asynchronous Apify client for interacting with the Apify API."""
         if not self._apify_client:
             self._apify_client = self.new_client()
         return self._apify_client
 
     @cached_property
     def configuration(self) -> Configuration:
-        """The Configuration instance the Actor instance uses."""
+        """Actor configuration, uses the default instance if not explicitly set."""
         if self._configuration:
             return self._configuration
 
@@ -214,8 +337,11 @@ class _ActorType:
 
     @cached_property
     def event_manager(self) -> EventManager:
-        """The EventManager instance the Actor instance uses."""
-        return (
+        """Manages Apify platform events.
+
+        It uses `ApifyEventManager` on the Apify platform and `LocalEventManager` otherwise.
+        """
+        event_manager = (
             ApifyEventManager(
                 configuration=self.configuration,
                 persist_state_interval=self.configuration.persist_state_interval,
@@ -226,19 +352,16 @@ class _ActorType:
                 persist_state_interval=self.configuration.persist_state_interval,
             )
         )
+        service_locator.set_event_manager(event_manager)
+        return event_manager
 
-    @property
-    def log(self) -> logging.Logger:
-        """The logging.Logger instance the Actor uses."""
-        return logger
-
-    def _raise_if_not_initialized(self) -> None:
-        if not self._is_initialized:
-            raise RuntimeError('The Actor was not initialized!')
+    @cached_property
+    def _charging_manager_implementation(self) -> ChargingManagerImplementation:
+        return ChargingManagerImplementation(self.configuration, self.apify_client)
 
     @cached_property
     def _storage_client(self) -> SmartApifyStorageClient:
-        """Storage client used by the actor.
+        """Storage client used by the Actor.
 
         Depending on the initialization of the service locator the client can be created in different ways.
         """
@@ -250,7 +373,7 @@ class _ActorType:
             service_locator.set_storage_client(implicit_storage_client)
         except ServiceConflictError:
             self.log.debug(
-                'Storage client in service locator was set explicitly before Actor.init was called.'
+                'Storage client in service locator was set explicitly before Actor.init was called. '
                 'Using the existing storage client as implicit storage client for the Actor.'
             )
         else:
@@ -270,100 +393,35 @@ class _ActorType:
         )
 
     async def init(self) -> None:
-        """Initialize the Actor instance.
-
-        This initializes the Actor instance. It configures the right storage client based on whether the Actor is
-        running locally or on the Apify platform, it initializes the event manager for processing Actor events,
-        and starts an interval for regularly sending `PERSIST_STATE` events, so that the Actor can regularly persist
-        its state in response to these events.
+        """Initialize the Actor without using context-manager syntax.
 
-        This method should be called immediately before performing any additional Actor actions, and it should be
-        called only once.
+        Equivalent to `await Actor.__aenter__()`.
         """
-        self.log.info('Initializing Actor...')
-        if self._configuration:
-            # Set explicitly the configuration in the service locator
-            service_locator.set_configuration(self.configuration)
-        else:
-            # Ensure that the configuration (cached property) is set
-            _ = self.configuration
-
-        if self._is_initialized:
-            raise RuntimeError('The Actor was already initialized!')
-
-        if _ActorType._is_any_instance_initialized:
-            self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care')
-
-        # Make sure that the currently initialized instance is also available through the global `Actor` proxy
-        cast('Proxy', Actor).__wrapped__ = self
-
-        self._is_exiting = False
-        self._was_final_persist_state_emitted = False
-
-        self.log.debug(f'Storage client set to {self._storage_client}')
-
-        service_locator.set_event_manager(self.event_manager)
-
-        # The logging configuration has to be called after all service_locator set methods.
-        if self._configure_logging:
-            _configure_logging()
-
-        self.log.info('System info', extra=get_system_info())
-
-        await self.event_manager.__aenter__()
-        self.log.debug('Event manager initialized')
-
-        await self._charging_manager_implementation.__aenter__()
-        self.log.debug('Charging manager initialized')
-
-        self._is_initialized = True
-        _ActorType._is_any_instance_initialized = True
+        await self.__aenter__()
 
     async def exit(
         self,
         *,
         exit_code: int = 0,
-        event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT,
         status_message: str | None = None,
+        event_listeners_timeout: timedelta | None = EVENT_LISTENERS_TIMEOUT,
         cleanup_timeout: timedelta = timedelta(seconds=30),
     ) -> None:
-        """Exit the Actor instance.
+        """Exit the Actor without using context-manager syntax.
 
-        This stops the Actor instance. It cancels all the intervals for regularly sending `PERSIST_STATE` events,
-        sends a final `PERSIST_STATE` event, waits for all the event listeners to finish, and stops the event manager.
+        Equivalent to `await Actor.__aexit__()`.
 
         Args:
-            exit_code: The exit code with which the Actor should fail (defaults to `0`).
-            event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting.
-            status_message: The final status message that the Actor should display.
-            cleanup_timeout: How long we should wait for event listeners.
+            exit_code: The exit code the Actor should use when exiting.
+            status_message: Final status message to display upon Actor termination.
+            event_listeners_timeout: Maximum time to wait for Actor event listeners to complete before exiting.
+            cleanup_timeout: Maximum time to wait for cleanup tasks to finish.
         """
-        self._raise_if_not_initialized()
-
-        self._is_exiting = True
-
-        exit_code = maybe_extract_enum_member_value(exit_code)
-
-        self.log.info('Exiting Actor', extra={'exit_code': exit_code})
-
-        async def finalize() -> None:
-            if status_message is not None:
-                await self.set_status_message(status_message, is_terminal=True)
-
-            # Sleep for a bit so that the listeners have a chance to trigger
-            await asyncio.sleep(0.1)
-
-            if event_listeners_timeout:
-                await self.event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
-
-            await self.event_manager.__aexit__(None, None, None)
-            await self._charging_manager_implementation.__aexit__(None, None, None)
-
-        await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
-        self._is_initialized = False
-
-        if self._exit_process:
-            sys.exit(exit_code)
+        self.exit_code = exit_code
+        self.status_message = status_message
+        self._event_listeners_timeout = event_listeners_timeout
+        self._cleanup_timeout = cleanup_timeout
+        await self.__aexit__(None, None, None)
 
     async def fail(
         self,
@@ -372,23 +430,24 @@ class _ActorType:
         exception: BaseException | None = None,
         status_message: str | None = None,
     ) -> None:
-        """Fail the Actor instance.
+        """Fail the Actor instance without using context-manager syntax.
 
-        This performs all the same steps as Actor.exit(), but it additionally sets the exit code to `1` (by default).
+        Equivalent to setting the `self.exit_code` and `self.status_message` properties and using
+        `await Actor.__aexit__()`.
 
         Args:
             exit_code: The exit code with which the Actor should fail (defaults to `1`).
             exception: The exception with which the Actor failed.
             status_message: The final status message that the Actor should display.
         """
-        self._raise_if_not_initialized()
-
-        # In IPython, we don't run `sys.exit()` during Actor exits,
-        # so the exception traceback will be printed on its own
-        if exception and not is_running_in_ipython():
-            self.log.exception('Actor failed with an exception', exc_info=exception)
+        self.exit_code = exit_code
+        self.status_message = status_message
 
-        await self.exit(exit_code=exit_code, status_message=status_message)
+        await self.__aexit__(
+            exc_type=type(exception) if exception else None,
+            exc_value=exception,
+            exc_traceback=exception.__traceback__ if exception else None,
+        )
 
     def new_client(
         self,
@@ -626,10 +685,6 @@ class _ActorType:
         self._raise_if_not_initialized()
         return self._charging_manager_implementation
 
-    @cached_property
-    def _charging_manager_implementation(self) -> ChargingManagerImplementation:
-        return ChargingManagerImplementation(self.configuration, self.apify_client)
-
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
 
@@ -822,18 +877,6 @@ class _ActorType:
 
         return ActorRun.model_validate(api_result)
 
-    def _get_remaining_time(self) -> timedelta | None:
-        """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform."""
-        if self.is_at_home() and self.configuration.timeout_at:
-            return self.configuration.timeout_at - datetime.now(tz=timezone.utc)
-
-        self.log.warning(
-            'Returning `None` instead of remaining time. Using `RemainingTime` argument is only possible when the Actor'
-            ' is running on the Apify platform and when the timeout for the Actor run is set. '
-            f'{self.is_at_home()=}, {self.configuration.timeout_at=}'
-        )
-        return None
-
     async def abort(
         self,
         run_id: str,
@@ -1242,6 +1285,10 @@ class _ActorType:
 
         return proxy_configuration
 
+    def _raise_if_not_initialized(self) -> None:
+        if not self._is_initialized:
+            raise RuntimeError('The Actor was not initialized!')
+
     def _get_default_exit_process(self) -> bool:
         """Return False for IPython, Pytest, and Scrapy environments, True otherwise."""
         if is_running_in_ipython():
@@ -1262,6 +1309,18 @@ class _ActorType:
 
         return True
 
+    def _get_remaining_time(self) -> timedelta | None:
+        """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform."""
+        if self.is_at_home() and self.configuration.timeout_at:
+            return self.configuration.timeout_at - datetime.now(tz=timezone.utc)
+
+        self.log.warning(
+            'Returning `None` instead of remaining time. Using `RemainingTime` argument is only possible when the Actor'
+            ' is running on the Apify platform and when the timeout for the Actor run is set. '
+            f'{self.is_at_home()=}, {self.configuration.timeout_at=}'
+        )
+        return None
+
 
 Actor = cast('_ActorType', Proxy(_ActorType))
 """The entry point of the SDK, through which all the Actor operations should be done."""

apify/_configuration.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from datetime import datetime, timedelta
 from decimal import Decimal
 from logging import getLogger
+from pathlib import Path
 from typing import Annotated, Any
 
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
@@ -421,6 +422,14 @@ class Configuration(CrawleeConfiguration):
             logger.warning('Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.')
         return self
 
+    @property
+    def canonical_input_key(self) -> str:
+        return str(Path(self.input_key).with_suffix('.json'))
+
+    @property
+    def input_key_candidates(self) -> set[str]:
+        return {self.input_key, self.canonical_input_key, Path(self.canonical_input_key).stem}
+
     @classmethod
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.

apify/scrapy/middlewares/apify_proxy.py

@@ -49,7 +49,7 @@ class ApifyHttpProxyMiddleware:
         if proxy_settings is None:
             Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing '
-                ' in the Actor input.'
+                'in the Actor input.'
             )
             raise NotConfigured
 

apify/storage_clients/_apify/_request_queue_client.py

@@ -12,8 +12,8 @@ from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest
 from crawlee.storages import RequestQueue
 
 from ._models import ApifyRequestQueueMetadata, RequestQueueStats
-from ._request_queue_shared_client import _ApifyRequestQueueSharedClient
-from ._request_queue_single_client import _ApifyRequestQueueSingleClient
+from ._request_queue_shared_client import ApifyRequestQueueSharedClient
+from ._request_queue_single_client import ApifyRequestQueueSingleClient
 from ._utils import AliasResolver
 
 if TYPE_CHECKING:
@@ -47,14 +47,14 @@ class ApifyRequestQueueClient(RequestQueueClient):
         self._api_client = api_client
         """The Apify request queue client for API operations."""
 
-        self._implementation: _ApifyRequestQueueSingleClient | _ApifyRequestQueueSharedClient
+        self._implementation: ApifyRequestQueueSingleClient | ApifyRequestQueueSharedClient
         """Internal implementation used to communicate with the Apify platform based Request Queue."""
         if access == 'single':
-            self._implementation = _ApifyRequestQueueSingleClient(
+            self._implementation = ApifyRequestQueueSingleClient(
                 api_client=self._api_client, metadata=metadata, cache_size=self._MAX_CACHED_REQUESTS
             )
         elif access == 'shared':
-            self._implementation = _ApifyRequestQueueSharedClient(
+            self._implementation = ApifyRequestQueueSharedClient(
                 api_client=self._api_client,
                 metadata=metadata,
                 cache_size=self._MAX_CACHED_REQUESTS,

apify/storage_clients/_apify/_request_queue_shared_client.py

@@ -23,7 +23,7 @@ if TYPE_CHECKING:
 logger = getLogger(__name__)
 
 
-class _ApifyRequestQueueSharedClient:
+class ApifyRequestQueueSharedClient:
     """An Apify platform implementation of the request queue client.
 
     This implementation supports multiple producers and multiple consumers scenario.
@@ -106,23 +106,19 @@ class _ApifyRequestQueueSharedClient:
                 # It could have been handled by another client in the meantime, so cached information about
                 # `request.was_already_handled` is not reliable.
                 already_present_requests.append(
-                    ProcessedRequest.model_validate(
-                        {
-                            'uniqueKey': request.unique_key,
-                            'wasAlreadyPresent': True,
-                            'wasAlreadyHandled': request.was_already_handled,
-                        }
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=request.was_already_handled,
                     )
                 )
 
             else:
                 # Add new request to the cache.
-                processed_request = ProcessedRequest.model_validate(
-                    {
-                        'uniqueKey': request.unique_key,
-                        'wasAlreadyPresent': True,
-                        'wasAlreadyHandled': request.was_already_handled,
-                    }
+                processed_request = ProcessedRequest(
+                    unique_key=request.unique_key,
+                    was_already_present=True,
+                    was_already_handled=request.was_already_handled,
                 )
                 self._cache_request(
                     request.unique_key,

apify/storage_clients/_apify/_request_queue_single_client.py

@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 logger = getLogger(__name__)
 
 
-class _ApifyRequestQueueSingleClient:
+class ApifyRequestQueueSingleClient:
     """An Apify platform implementation of the request queue client with limited capability.
 
     This client is designed to use as little resources as possible, but has to be used in constrained context.
@@ -108,23 +108,19 @@ class _ApifyRequestQueueSingleClient:
             # Check if request is known to be already handled (it has to be present as well.)
             if request.unique_key in self._requests_already_handled:
                 already_present_requests.append(
-                    ProcessedRequest.model_validate(
-                        {
-                            'uniqueKey': request.unique_key,
-                            'wasAlreadyPresent': True,
-                            'wasAlreadyHandled': True,
-                        }
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=True,
                     )
                 )
             # Check if request is known to be already present, but unhandled
             elif self._requests_cache.get(request.unique_key):
                 already_present_requests.append(
-                    ProcessedRequest.model_validate(
-                        {
-                            'uniqueKey': request.unique_key,
-                            'wasAlreadyPresent': True,
-                            'wasAlreadyHandled': request.was_already_handled,
-                        }
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=request.was_already_handled,
                     )
                 )
             else:
@@ -158,8 +154,9 @@ class _ApifyRequestQueueSingleClient:
                 self._requests_cache.pop(unprocessed_request.unique_key, None)
 
         else:
-            api_response = AddRequestsResponse.model_validate(
-                {'unprocessedRequests': [], 'processedRequests': already_present_requests}
+            api_response = AddRequestsResponse(
+                unprocessed_requests=[],
+                processed_requests=already_present_requests,
             )
 
         # Update assumed total count for newly added requests.
@@ -236,20 +233,39 @@ class _ApifyRequestQueueSingleClient:
 
         # Update the cached data
         for request_data in response.get('items', []):
+            # Due to https://github.com/apify/apify-core/blob/v0.1377.0/src/api/src/lib/request_queues/request_queue.ts#L53,
+            # the list_head endpoint may return truncated fields for long requests (e.g., long URLs or unique keys).
+            # If truncation is detected, fetch the full request data by its ID from the API.
+            # This is a temporary workaround - the caching will be refactored to use request IDs instead of unique keys.
+            # See https://github.com/apify/apify-sdk-python/issues/630 for details.
+            if '[truncated]' in request_data['uniqueKey'] or '[truncated]' in request_data['url']:
+                request_data = await self._api_client.get_request(request_id=request_data['id'])  # noqa: PLW2901
+
             request = Request.model_validate(request_data)
 
             if request.unique_key in self._requests_in_progress:
                 # Ignore requests that are already in progress, we will not process them again.
                 continue
+
             if request.was_already_handled:
                 # Do not cache fully handled requests, we do not need them. Just cache their unique_key.
                 self._requests_already_handled.add(request.unique_key)
             else:
                 # Only fetch the request if we do not know it yet.
                 if request.unique_key not in self._requests_cache:
-                    request = Request.model_validate(
-                        await self._api_client.get_request(unique_key_to_request_id(request.unique_key))
-                    )
+                    request_id = unique_key_to_request_id(request.unique_key)
+
+                    if request_data is not None and request_id != request_data['id']:
+                        logger.warning(
+                            f'Request ID mismatch: {request_id} != {request_data["id"]}, '
+                            'this may cause unexpected behavior.'
+                        )
+
+                    # See https://github.com/apify/apify-sdk-python/issues/630 for details.
+                    if '[truncated]' not in request.unique_key:
+                        request_data = await self._api_client.get_request(request_id=request_id)  # noqa: PLW2901
+                        request = Request.model_validate(request_data)
+
                     self._requests_cache[request.unique_key] = request
 
                 # Add new requests to the end of the head, unless already present in head

apify/storage_clients/_file_system/_key_value_store_client.py

@@ -1,14 +1,19 @@
 import asyncio
 import json
-from pathlib import Path
+import logging
 
-from typing_extensions import override
+from more_itertools import flatten
+from typing_extensions import Self, override
 
 from crawlee._consts import METADATA_FILENAME
+from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient
+from crawlee.storage_clients.models import KeyValueStoreRecord
 
 from apify._configuration import Configuration
 
+logger = logging.getLogger(__name__)
+
 
 class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
     """Apify-specific implementation of the `FileSystemKeyValueStoreClient`.
@@ -17,6 +22,22 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
     directory, except for the metadata file and the `INPUT.json` file.
     """
 
+    @override
+    @classmethod
+    async def open(
+        cls,
+        *,
+        id: str | None,
+        name: str | None,
+        alias: str | None,
+        configuration: CrawleeConfiguration,
+    ) -> Self:
+        client = await super().open(id=id, name=name, alias=alias, configuration=configuration)
+
+        await client._sanitize_input_json_files()  # noqa: SLF001 - it's okay, this is a factory method
+
+        return client
+
     @override
     async def purge(self) -> None:
         """Purges the key-value store by deleting all its contents.
@@ -24,16 +45,16 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
         It deletes all files in the key-value store directory, except for the metadata file and
         the `INPUT.json` file. It also updates the metadata to reflect that the store has been purged.
         """
-        kvs_input_key = Configuration.get_global_configuration().input_key
-
-        # First try to find the alternative format of the input file and process it if it exists.
-        for file_path in self.path_to_kvs.glob('*'):
-            if file_path.name == f'{kvs_input_key}.json':
-                await self._process_input_json(file_path)
+        configuration = Configuration.get_global_configuration()
 
         async with self._lock:
+            files_to_keep = set(
+                flatten([key, f'{key}.{METADATA_FILENAME}'] for key in configuration.input_key_candidates)
+            )
+            files_to_keep.add(METADATA_FILENAME)
+
             for file_path in self.path_to_kvs.glob('*'):
-                if file_path.name in {METADATA_FILENAME, kvs_input_key, f'{kvs_input_key}.{METADATA_FILENAME}'}:
+                if file_path.name in files_to_keep:
                     continue
                 if file_path.is_file():
                     await asyncio.to_thread(file_path.unlink, missing_ok=True)
@@ -43,15 +64,40 @@ class ApifyFileSystemKeyValueStoreClient(FileSystemKeyValueStoreClient):
                 update_modified_at=True,
             )
 
-    async def _process_input_json(self, path: Path) -> None:
-        """Process simple input json file to format expected by the FileSystemKeyValueStoreClient.
+    async def _sanitize_input_json_files(self) -> None:
+        """Handle missing metadata for input files."""
+        configuration = Configuration.get_global_configuration()
+        alternative_keys = configuration.input_key_candidates - {configuration.canonical_input_key}
 
-        For example: INPUT.json -> INPUT, INPUT.json.metadata
-        """
-        try:
-            f = await asyncio.to_thread(path.open)
-            input_data = json.load(f)
-        finally:
-            f.close()
-        await asyncio.to_thread(path.unlink, missing_ok=True)
-        await self.set_value(key=path.stem, value=input_data)
+        if (self.path_to_kvs / configuration.canonical_input_key).exists():
+            # Refresh metadata to prevent inconsistencies
+            input_data = await asyncio.to_thread(
+                lambda: json.loads((self.path_to_kvs / configuration.canonical_input_key).read_text())
+            )
+            await self.set_value(key=configuration.canonical_input_key, value=input_data)
+
+            for alternative_key in alternative_keys:
+                if (alternative_input_file := self.path_to_kvs / alternative_key).exists():
+                    logger.warning(f'Redundant input file found: {alternative_input_file}')
+        else:
+            for alternative_key in alternative_keys:
+                alternative_input_file = self.path_to_kvs / alternative_key
+
+                # Only process files that actually exist
+                if alternative_input_file.exists():
+                    # Refresh metadata to prevent inconsistencies
+                    with alternative_input_file.open() as f:
+                        input_data = await asyncio.to_thread(lambda: json.load(f))
+                    await self.set_value(key=alternative_key, value=input_data)
+
+    @override
+    async def get_value(self, *, key: str) -> KeyValueStoreRecord | None:
+        configuration = Configuration.get_global_configuration()
+
+        if key in configuration.input_key_candidates:
+            for candidate in configuration.input_key_candidates:
+                value = await super().get_value(key=candidate)
+                if value is not None:
+                    return value
+
+        return await super().get_value(key=key)

{apify-3.0.1b1.dist-info → apify-3.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.0.1b1
+Version: 3.0.2
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -228,7 +228,7 @@ Requires-Python: >=3.10
 Requires-Dist: apify-client<3.0.0,>=2.0.0
 Requires-Dist: apify-shared<3.0.0,>=2.0.0
 Requires-Dist: cachetools>=5.5.0
-Requires-Dist: crawlee<2.0.0,>=1.0.0
+Requires-Dist: crawlee<2.0.0,>=1.0.2
 Requires-Dist: cryptography>=42.0.0
 Requires-Dist: impit>=0.6.1
 Requires-Dist: lazy-object-proxy>=1.11.0

{apify-3.0.1b1.dist-info → apify-3.0.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
-apify/_actor.py,sha256=DYHoyBAu6hDLs0BcTZL-IQveLK8gPTWvb6AgDnJc3EA,54755
+apify/_actor.py,sha256=kfrwD8gaeN4NcdNMD_Pj66agNh78jJjwMuNOuwLdo-E,57370
 apify/_charging.py,sha256=KjZ2DnEMS0Tt8ibizmmt0RwBq8FOAsD1z-hKFgdazcY,13143
-apify/_configuration.py,sha256=gq_UfWTgcP1_0kEMLhXVg33SgSxXjShbuzoXyCFfK0w,14682
+apify/_configuration.py,sha256=7ZHhgRp98kr35zx4k4EB2aImq7Dq1FJjPg7r5bucv_M,14984
 apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
 apify/_crypto.py,sha256=tqUs13QkemDtGzvU41pIA2HUEawpDlgzqbwKjm4I8kM,6852
 apify/_models.py,sha256=EzU-inWeJ7T5HNVYEwnYb79W-q4OAPhtrYctfRYzpTE,7848
@@ -27,7 +27,7 @@ apify/scrapy/utils.py,sha256=Ssfa-P9-g9XYP1suDce6dQ8ta7PfijiPoMl2iplE6Ow,2126
 apify/scrapy/extensions/__init__.py,sha256=cVQ8CCtOsJsRP28YKZWSUsi4FBwxI-yPJRNSXPFSa_o,98
 apify/scrapy/extensions/_httpcache.py,sha256=XIS9vFCcUtdSfeKAKnxRnI9dX_GMmX2Od8OPnOaDhQ0,8870
 apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
-apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
+apify/scrapy/middlewares/apify_proxy.py,sha256=EtR0958hGfBZ8MfwOIc_XyfBYPdMSkul-Ew4eYQCalM,5549
 apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
 apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
@@ -38,20 +38,20 @@ apify/storage_clients/_apify/__init__.py,sha256=mtbVDsxqWL3kx30elnh0kAn2kZ4s3BBs
 apify/storage_clients/_apify/_dataset_client.py,sha256=Bb3UwOaFkyuEY7tuBf8K46R4ZP_b1EaAkDOXOqwSoW8,12498
 apify/storage_clients/_apify/_key_value_store_client.py,sha256=42dARbLX2oeOW7uYYKkDyQbEriMuh55Mxh0SqvkOEGg,10529
 apify/storage_clients/_apify/_models.py,sha256=GEaN7Got1zIg42QPH36obHRWRDVNtzOkRuOWYRf9bFU,4572
-apify/storage_clients/_apify/_request_queue_client.py,sha256=QXCLdTBeNW8RKWnxQOE71KOpZ_lqvqisa89eeiWwZ38,14200
-apify/storage_clients/_apify/_request_queue_shared_client.py,sha256=CbvwcXRvfuBoy3wrQEdLX9_vKELPH_WhHQARP14audM,20709
-apify/storage_clients/_apify/_request_queue_single_client.py,sha256=6CRSyWZPbKQJy3i2JBrojVTnhTYIB3gE0CTymYjpkZA,16958
+apify/storage_clients/_apify/_request_queue_client.py,sha256=PUIVmGQxqFTkRxW9FIFWjT0OeDyAGt-ULlW-rdQDTyc,14194
+apify/storage_clients/_apify/_request_queue_shared_client.py,sha256=uxkuIG1rgCArgs6agldC9vmB2bgrIlNnm1I214Gf6WA,20550
+apify/storage_clients/_apify/_request_queue_single_client.py,sha256=EuORHJnFLC1YAT6ZfQj-ayrfSJNpU4_61r_7uDyvwgA,18092
 apify/storage_clients/_apify/_storage_client.py,sha256=hFl_PuX1UgOydBD6pieZ0u2NWbDmZV-i0qygKdsuHt4,4873
 apify/storage_clients/_apify/_utils.py,sha256=ywXoSM69amRokUZcshbAvQLIcSZq4L-bpYIGyeFxCGQ,7696
 apify/storage_clients/_apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/storage_clients/_file_system/__init__.py,sha256=rDbXatXV9wHKPhKTrXDzWnexhTm7sIJQWucMi-P-SD4,130
-apify/storage_clients/_file_system/_key_value_store_client.py,sha256=fnSJ1EIOPCGfcE6e5S3Tux9VbnMVLCJjugkaQoH_9yo,2267
+apify/storage_clients/_file_system/_key_value_store_client.py,sha256=gxM3ap67PnY80Rd7P3onPAf2pksYpU0LoAlJdayEMdU,4179
 apify/storage_clients/_file_system/_storage_client.py,sha256=rcwpKYlrWzvlSA2xoxftg-EZAi_iGZ3vOCbu0C5lKDE,1396
 apify/storage_clients/_smart_apify/__init__.py,sha256=614B2AaWY-dx6RQ6mod7VVR8gFh75-_jnq5BeDD7hSc,53
 apify/storage_clients/_smart_apify/_storage_client.py,sha256=GCPmVe_xWAFcO2Cuej4su4i97_d33Q9Ih_Sc5xW2Wa4,4674
 apify/storages/__init__.py,sha256=-9tEYJVabVs_eRVhUehxN58GH0UG8OfuGjGwuDieP2M,122
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-3.0.1b1.dist-info/METADATA,sha256=Qy-fnT_4BnuEpoIhk_Aa0vIl6GVQtkqkk8diacKkzA0,22582
-apify-3.0.1b1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-apify-3.0.1b1.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-3.0.1b1.dist-info/RECORD,,
+apify-3.0.2.dist-info/METADATA,sha256=YRyzVDZFQFOevC5s2rezm0kM-0OFXb2HarPNZlRHG9Y,22580
+apify-3.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+apify-3.0.2.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-3.0.2.dist-info/RECORD,,