apify 1.7.1b1__py3-none-any.whl → 2.2.1__py3-none-any.whl

apify/config.py

@@ -1,127 +0,0 @@
-from __future__ import annotations
-
-from apify_shared.consts import ActorEnvVars, ApifyEnvVars
-
-from apify._utils import fetch_and_parse_env_var
-
-
-class Configuration:
-    """A class for specifying the configuration of an actor.
-
-    Can be used either globally via `Configuration.get_global_configuration()`,
-    or it can be specific to each `Actor` instance on the `actor.config` property.
-    """
-
-    _default_instance: Configuration | None = None
-
-    def __init__(
-        self: Configuration,
-        *,
-        api_base_url: str | None = None,
-        api_public_base_url: str | None = None,
-        container_port: int | None = None,
-        container_url: str | None = None,
-        default_dataset_id: str | None = None,
-        default_key_value_store_id: str | None = None,
-        default_request_queue_id: str | None = None,
-        input_key: str | None = None,
-        max_used_cpu_ratio: float | None = None,
-        metamorph_after_sleep_millis: int | None = None,
-        persist_state_interval_millis: int | None = None,
-        persist_storage: bool | None = None,
-        proxy_hostname: str | None = None,
-        proxy_password: str | None = None,
-        proxy_port: int | None = None,
-        proxy_status_url: str | None = None,
-        purge_on_start: bool | None = None,
-        token: str | None = None,
-        system_info_interval_millis: int | None = None,
-    ) -> None:
-        """Create a `Configuration` instance.
-
-        All the parameters are loaded by default from environment variables when running on the Apify platform.
-        You can override them here in the Configuration constructor, which might be useful for local testing of your actors.
-
-        Args:
-            api_base_url (str, optional): The URL of the Apify API.
-                This is the URL actually used for connecting to the API, so it can contain an IP address when running in a container on the platform.
-            api_public_base_url (str, optional): The public URL of the Apify API.
-                This will always contain the public URL of the API, even when running in a container on the platform.
-                Useful for generating shareable URLs to key-value store records or datasets.
-            container_port (int, optional): The port on which the container can listen for HTTP requests.
-            container_url (str, optional): The URL on which the container can listen for HTTP requests.
-            default_dataset_id (str, optional): The ID of the default dataset for the actor.
-            default_key_value_store_id (str, optional): The ID of the default key-value store for the actor.
-            default_request_queue_id (str, optional): The ID of the default request queue for the actor.
-            input_key (str, optional): The key of the input record in the actor's default key-value store
-            max_used_cpu_ratio (float, optional): The CPU usage above which the SYSTEM_INFO event will report the CPU is overloaded.
-            metamorph_after_sleep_millis (int, optional): How long should the actor sleep after calling metamorph.
-            persist_state_interval_millis (int, optional): How often should the actor emit the PERSIST_STATE event.
-            persist_storage (bool, optional): Whether the actor should persist its used storages to the filesystem when running locally.
-            proxy_hostname (str, optional): The hostname of Apify Proxy.
-            proxy_password (str, optional): The password for Apify Proxy.
-            proxy_port (str, optional): The port of Apify Proxy.
-            proxy_status_url (str, optional): The URL on which the Apify Proxy status page is available.
-            purge_on_start (str, optional): Whether the actor should purge its default storages on startup, when running locally.
-            token (str, optional): The API token for the Apify API this actor should use.
-            system_info_interval_millis (str, optional): How often should the actor emit the SYSTEM_INFO event when running locally.
-        """
-        # TODO: Document all these members
-        # https://github.com/apify/apify-sdk-python/issues/147
-        self.actor_build_id = fetch_and_parse_env_var(ActorEnvVars.BUILD_ID)
-        self.actor_build_number = fetch_and_parse_env_var(ActorEnvVars.BUILD_NUMBER)
-        self.actor_events_ws_url = fetch_and_parse_env_var(ActorEnvVars.EVENTS_WEBSOCKET_URL)
-        self.actor_id = fetch_and_parse_env_var(ActorEnvVars.ID)
-        self.actor_run_id = fetch_and_parse_env_var(ActorEnvVars.RUN_ID)
-        self.actor_task_id = fetch_and_parse_env_var(ActorEnvVars.TASK_ID)
-        self.api_base_url = api_base_url or fetch_and_parse_env_var(ApifyEnvVars.API_BASE_URL, 'https://api.apify.com')
-        self.api_public_base_url = api_public_base_url or fetch_and_parse_env_var(ApifyEnvVars.API_PUBLIC_BASE_URL, 'https://api.apify.com')
-        self.chrome_executable_path = fetch_and_parse_env_var(ApifyEnvVars.CHROME_EXECUTABLE_PATH)
-        self.container_port = container_port or fetch_and_parse_env_var(ActorEnvVars.WEB_SERVER_PORT, 4321)
-        self.container_url = container_url or fetch_and_parse_env_var(ActorEnvVars.WEB_SERVER_URL, 'http://localhost:4321')
-        self.dedicated_cpus = fetch_and_parse_env_var(ApifyEnvVars.DEDICATED_CPUS)
-        self.default_browser_path = fetch_and_parse_env_var(ApifyEnvVars.DEFAULT_BROWSER_PATH)
-        self.default_dataset_id = default_dataset_id or fetch_and_parse_env_var(ActorEnvVars.DEFAULT_DATASET_ID, 'default')
-        self.default_key_value_store_id = default_key_value_store_id or fetch_and_parse_env_var(ActorEnvVars.DEFAULT_KEY_VALUE_STORE_ID, 'default')
-        self.default_request_queue_id = default_request_queue_id or fetch_and_parse_env_var(ActorEnvVars.DEFAULT_REQUEST_QUEUE_ID, 'default')
-        self.disable_browser_sandbox = fetch_and_parse_env_var(ApifyEnvVars.DISABLE_BROWSER_SANDBOX, default=False)
-        self.headless = fetch_and_parse_env_var(ApifyEnvVars.HEADLESS, default=True)
-        self.input_key = input_key or fetch_and_parse_env_var(ActorEnvVars.INPUT_KEY, 'INPUT')
-        self.input_secrets_private_key_file = fetch_and_parse_env_var(ApifyEnvVars.INPUT_SECRETS_PRIVATE_KEY_FILE)
-        self.input_secrets_private_key_passphrase = fetch_and_parse_env_var(ApifyEnvVars.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE)
-        self.is_at_home = fetch_and_parse_env_var(ApifyEnvVars.IS_AT_HOME, default=False)
-        self.max_used_cpu_ratio = max_used_cpu_ratio or fetch_and_parse_env_var(ApifyEnvVars.MAX_USED_CPU_RATIO, 0.95)
-        self.memory_mbytes = fetch_and_parse_env_var(ActorEnvVars.MEMORY_MBYTES)
-        self.meta_origin = fetch_and_parse_env_var(ApifyEnvVars.META_ORIGIN)
-        self.metamorph_after_sleep_millis = metamorph_after_sleep_millis or fetch_and_parse_env_var(ApifyEnvVars.METAMORPH_AFTER_SLEEP_MILLIS, 300000)
-        self.persist_state_interval_millis = persist_state_interval_millis or fetch_and_parse_env_var(
-            ApifyEnvVars.PERSIST_STATE_INTERVAL_MILLIS, 60000
-        )
-        self.persist_storage = persist_storage or fetch_and_parse_env_var(ApifyEnvVars.PERSIST_STORAGE, default=True)
-        self.proxy_hostname = proxy_hostname or fetch_and_parse_env_var(ApifyEnvVars.PROXY_HOSTNAME, 'proxy.apify.com')
-        self.proxy_password = proxy_password or fetch_and_parse_env_var(ApifyEnvVars.PROXY_PASSWORD)
-        self.proxy_port = proxy_port or fetch_and_parse_env_var(ApifyEnvVars.PROXY_PORT, 8000)
-        self.proxy_status_url = proxy_status_url or fetch_and_parse_env_var(ApifyEnvVars.PROXY_STATUS_URL, 'http://proxy.apify.com')
-        self.purge_on_start = purge_on_start or fetch_and_parse_env_var(ApifyEnvVars.PURGE_ON_START, default=False)
-        self.started_at = fetch_and_parse_env_var(ActorEnvVars.STARTED_AT)
-        self.timeout_at = fetch_and_parse_env_var(ActorEnvVars.TIMEOUT_AT)
-        self.token = token or fetch_and_parse_env_var(ApifyEnvVars.TOKEN)
-        self.user_id = fetch_and_parse_env_var(ApifyEnvVars.USER_ID)
-        self.xvfb = fetch_and_parse_env_var(ApifyEnvVars.XVFB, default=False)
-        self.system_info_interval_millis = system_info_interval_millis or fetch_and_parse_env_var(ApifyEnvVars.SYSTEM_INFO_INTERVAL_MILLIS, 60000)
-
-    @classmethod
-    def _get_default_instance(cls: type[Configuration]) -> Configuration:
-        if cls._default_instance is None:
-            cls._default_instance = cls()
-
-        return cls._default_instance
-
-    @classmethod
-    def get_global_configuration(cls: type[Configuration]) -> Configuration:
-        """Retrive the global configuration.
-
-        The global configuration applies when you call actor methods via their static versions, e.g. `Actor.init()`.
-        Also accessible via `Actor.config`.
-        """
-        return cls._get_default_instance()

apify/consts.py

@@ -1,67 +0,0 @@
-from __future__ import annotations
-
-import re
-import warnings
-from enum import Enum
-from typing import Any
-
-from apify_shared.consts import BOOL_ENV_VARS as _BOOL_ENV_VARS  # noqa: F401
-from apify_shared.consts import DATETIME_ENV_VARS as _DATETIME_ENV_VARS  # noqa: F401
-from apify_shared.consts import FLOAT_ENV_VARS as _FLOAT_ENV_VARS  # noqa: F401
-from apify_shared.consts import INTEGER_ENV_VARS as _INTEGER_ENV_VARS  # noqa: F401
-from apify_shared.consts import STRING_ENV_VARS as _STRING_ENV_VARS  # noqa: F401
-from apify_shared.consts import ActorEventTypes as _ActorEventTypes  # noqa: F401
-from apify_shared.consts import ActorExitCodes as _ActorExitCodes  # noqa: F401
-from apify_shared.consts import ApifyEnvVars as _ApifyEnvVars  # noqa: F401
-
-DEPRECATED_NAMES = [
-    'BOOL_ENV_VARS',
-    'DATETIME_ENV_VARS',
-    'FLOAT_ENV_VARS',
-    'INTEGER_ENV_VARS',
-    'STRING_ENV_VARS',
-    'ActorEventTypes',
-    'ActorExitCodes',
-    'ApifyEnvVars',
-]
-
-
-# The following piece of code is highly inspired by the example in https://peps.python.org/pep-0562.
-# The else branch is missing intentionally! Check the following discussion for details:
-# https://github.com/apify/apify-client-python/pull/132#discussion_r1277294315.
-def __getattr__(name: str) -> Any:
-    if name in DEPRECATED_NAMES:
-        warnings.warn(
-            (
-                f'Importing "{name}" from "apify_client.consts" is deprecated and will be removed in the future. '
-                'Please use "apify_shared" library instead.'
-            ),
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-        return globals()[f'_{name}']
-    raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
-
-
-class StorageTypes(str, Enum):
-    """Possible Apify storage types."""
-
-    DATASET = 'Dataset'
-    KEY_VALUE_STORE = 'Key-value store'
-    REQUEST_QUEUE = 'Request queue'
-
-
-DEFAULT_API_PARAM_LIMIT = 1000
-
-REQUEST_ID_LENGTH = 15
-
-REQUEST_QUEUE_HEAD_MAX_LIMIT = 1000
-
-EVENT_LISTENERS_TIMEOUT_SECS = 5
-
-BASE64_REGEXP = '[-A-Za-z0-9+/]*={0,3}'
-ENCRYPTED_INPUT_VALUE_PREFIX = 'ENCRYPTED_VALUE'
-ENCRYPTED_INPUT_VALUE_REGEXP = re.compile(f'^{ENCRYPTED_INPUT_VALUE_PREFIX}:({BASE64_REGEXP}):({BASE64_REGEXP})$')
-
-# 9MB
-MAX_PAYLOAD_SIZE_BYTES = 9437184

apify/event_manager.py

@@ -1,236 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import inspect
-import json
-from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Callable, Coroutine, Union
-
-import websockets.client
-from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value, parse_date_fields
-from pyee.asyncio import AsyncIOEventEmitter
-
-from apify.log import logger
-
-if TYPE_CHECKING:
-    from apify_shared.consts import ActorEventTypes
-
-    from apify.config import Configuration
-
-ListenerType = Union[Callable[[], None], Callable[[Any], None], Callable[[], Coroutine[Any, Any, None]], Callable[[Any], Coroutine[Any, Any, None]]]
-
-
-@ignore_docs
-class EventManager:
-    """A class for managing actor events.
-
-    You shouldn't use this class directly,
-    but instead use it via the `Actor.on()` and `Actor.off()` methods.
-    """
-
-    _platform_events_websocket: websockets.client.WebSocketClientProtocol | None = None
-    _process_platform_messages_task: asyncio.Task | None = None
-    _send_persist_state_interval_task: asyncio.Task | None = None
-    _send_system_info_interval_task: asyncio.Task | None = None
-    _listener_tasks: set[asyncio.Task]
-    _listeners_to_wrappers: dict[ActorEventTypes, dict[Callable, list[Callable]]]
-    _connected_to_platform_websocket: asyncio.Future | None = None
-
-    def __init__(self: EventManager, config: Configuration) -> None:
-        """Create an instance of the EventManager.
-
-        Args:
-            config (Configuration): The actor configuration to be used in this event manager.
-        """
-        self._config = config
-        self._event_emitter = AsyncIOEventEmitter()
-        self._initialized = False
-        self._listener_tasks = set()
-        self._listeners_to_wrappers = defaultdict(lambda: defaultdict(list))
-
-    async def init(self: EventManager) -> None:
-        """Initialize the event manager.
-
-        When running this on the Apify Platform, this will start processing events
-        send by the platform to the events websocket and emitting them as events
-        that can be listened to by the `Actor.on()` method.
-        """
-        if self._initialized:
-            raise RuntimeError('EventManager was already initialized!')
-
-        # Run tasks but don't await them
-        if self._config.actor_events_ws_url:
-            self._connected_to_platform_websocket = asyncio.Future()
-            self._process_platform_messages_task = asyncio.create_task(self._process_platform_messages())
-            is_connected = await self._connected_to_platform_websocket
-            if not is_connected:
-                raise RuntimeError('Error connecting to platform events websocket!')
-        else:
-            logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
-
-        self._initialized = True
-
-    async def close(self: EventManager, event_listeners_timeout_secs: float | None = None) -> None:
-        """Initialize the event manager.
-
-        This will stop listening for the platform events,
-        and it will wait for all the event listeners to finish.
-
-        Args:
-            event_listeners_timeout_secs (float, optional): Optional timeout after which the pending event listeners are canceled.
-        """
-        if not self._initialized:
-            raise RuntimeError('EventManager was not initialized!')
-
-        if self._platform_events_websocket:
-            await self._platform_events_websocket.close()
-
-        if self._process_platform_messages_task:
-            await self._process_platform_messages_task
-
-        await self.wait_for_all_listeners_to_complete(timeout_secs=event_listeners_timeout_secs)
-
-        self._event_emitter.remove_all_listeners()
-
-        self._initialized = False
-
-    def on(self: EventManager, event_name: ActorEventTypes, listener: ListenerType) -> Callable:
-        """Add an event listener to the event manager.
-
-        Args:
-            event_name (ActorEventTypes): The actor event for which to listen to.
-            listener (Callable): The function which is to be called when the event is emitted (can be async).
-                Must accept either zero or one arguments (the first argument will be the event data).
-        """
-        if not self._initialized:
-            raise RuntimeError('EventManager was not initialized!')
-
-        # Detect whether the listener will accept the event_data argument
-        try:
-            signature = inspect.signature(listener)
-        except (ValueError, TypeError):
-            # If we can't determine the listener argument count (e.g. for the built-in `print` function),
-            # let's assume the listener will accept the argument
-            listener_argument_count = 1
-        else:
-            try:
-                dummy_event_data: dict = {}
-                signature.bind(dummy_event_data)
-                listener_argument_count = 1
-            except TypeError:
-                try:
-                    signature.bind()
-                    listener_argument_count = 0
-                except TypeError as err:
-                    raise ValueError('The "listener" argument must be a callable which accepts 0 or 1 arguments!') from err
-
-        event_name = maybe_extract_enum_member_value(event_name)
-
-        async def inner_wrapper(event_data: Any) -> None:
-            if inspect.iscoroutinefunction(listener):
-                if listener_argument_count == 0:
-                    await listener()
-                else:
-                    await listener(event_data)
-            elif listener_argument_count == 0:
-                listener()  # type: ignore[call-arg]
-            else:
-                listener(event_data)  # type: ignore[call-arg]
-
-        async def outer_wrapper(event_data: Any) -> None:
-            listener_task = asyncio.create_task(inner_wrapper(event_data))
-            self._listener_tasks.add(listener_task)
-            try:
-                await listener_task
-            except asyncio.CancelledError:
-                raise
-            except Exception:
-                # We need to swallow the exception and just log it here, since it could break the event emitter otherwise
-                logger.exception('Exception in event listener', extra={'event_name': event_name, 'listener_name': listener.__name__})
-            finally:
-                self._listener_tasks.remove(listener_task)
-
-        self._listeners_to_wrappers[event_name][listener].append(outer_wrapper)
-
-        return self._event_emitter.add_listener(event_name, outer_wrapper)
-
-    def off(self: EventManager, event_name: ActorEventTypes, listener: Callable | None = None) -> None:
-        """Remove a listener, or all listeners, from an actor event.
-
-        Args:
-            event_name (ActorEventTypes): The actor event for which to remove listeners.
-            listener (Callable, optional): The listener which is supposed to be removed. If not passed, all listeners of this event are removed.
-        """
-        if not self._initialized:
-            raise RuntimeError('EventManager was not initialized!')
-
-        event_name = maybe_extract_enum_member_value(event_name)
-
-        if listener:
-            for listener_wrapper in self._listeners_to_wrappers[event_name][listener]:
-                self._event_emitter.remove_listener(event_name, listener_wrapper)
-            self._listeners_to_wrappers[event_name][listener] = []
-        else:
-            self._listeners_to_wrappers[event_name] = defaultdict(list)
-            self._event_emitter.remove_all_listeners(event_name)
-
-    def emit(self: EventManager, event_name: ActorEventTypes, data: Any) -> None:
-        """Emit an actor event manually.
-
-        Args:
-            event_name (ActorEventTypes): The actor event which should be emitted.
-            data (Any): The data that should be emitted with the event.
-        """
-        event_name = maybe_extract_enum_member_value(event_name)
-
-        self._event_emitter.emit(event_name, data)
-
-    async def wait_for_all_listeners_to_complete(self: EventManager, *, timeout_secs: float | None = None) -> None:
-        """Wait for all event listeners which are currently being executed to complete.
-
-        Args:
-            timeout_secs (float, optional): Timeout for the wait. If the event listeners don't finish until the timeout, they will be canceled.
-        """
-
-        async def _wait_for_listeners() -> None:
-            results = await asyncio.gather(*self._listener_tasks, return_exceptions=True)
-            for result in results:
-                if result is Exception:
-                    logger.exception('Event manager encountered an exception in one of the event listeners', exc_info=result)
-
-        if timeout_secs:
-            _, pending = await asyncio.wait([asyncio.create_task(_wait_for_listeners())], timeout=timeout_secs)
-            if pending:
-                logger.warning('Timed out waiting for event listeners to complete, unfinished event listeners will be canceled')
-                for pending_task in pending:
-                    pending_task.cancel()
-                    with contextlib.suppress(asyncio.CancelledError):
-                        await pending_task
-        else:
-            await _wait_for_listeners()
-
-    async def _process_platform_messages(self: EventManager) -> None:
-        # This should be called only on the platform, where we have the ACTOR_EVENTS_WS_URL configured
-        assert self._config.actor_events_ws_url is not None  # noqa: S101
-        assert self._connected_to_platform_websocket is not None  # noqa: S101
-
-        try:
-            async with websockets.client.connect(self._config.actor_events_ws_url) as websocket:
-                self._platform_events_websocket = websocket
-                self._connected_to_platform_websocket.set_result(True)
-                async for message in websocket:
-                    try:
-                        parsed_message = json.loads(message)
-                        assert isinstance(parsed_message, dict)  # noqa: S101
-                        parsed_message = parse_date_fields(parsed_message)
-                        event_name = parsed_message['name']
-                        event_data = parsed_message.get('data')  # 'data' can be missing
-
-                        self._event_emitter.emit(event_name, event_data)
-
-                    except Exception:
-                        logger.exception('Cannot parse actor event', extra={'message': message})
-        except Exception:
-            logger.exception('Error in websocket connection')
-            self._connected_to_platform_websocket.set_result(False)