apify 1.7.0b1__py3-none-any.whl → 2.2.0b14__py3-none-any.whl

apify/_configuration.py

@@ -0,0 +1,370 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from logging import getLogger
+from typing import Annotated, Any
+
+from pydantic import AliasChoices, BeforeValidator, Field
+from typing_extensions import deprecated
+
+from crawlee._utils.models import timedelta_ms
+from crawlee._utils.urls import validate_http_url
+from crawlee.configuration import Configuration as CrawleeConfiguration
+
+from apify._utils import docs_group
+
+logger = getLogger(__name__)
+
+
+def _transform_to_list(value: Any) -> list[str] | None:
+    if value is None:
+        return None
+    if not value:
+        return []
+    return value if isinstance(value, list) else str(value).split(',')
+
+
+@docs_group('Classes')
+class Configuration(CrawleeConfiguration):
+    """A class for specifying the configuration of an Actor.
+
+    Can be used either globally via `Configuration.get_global_configuration()`,
+    or it can be specific to each `Actor` instance on the `actor.config` property.
+    """
+
+    actor_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_id',
+                'apify_actor_id',
+                'apify_act_id',
+            ),
+            description='ID of the Actor',
+        ),
+    ] = None
+
+    actor_full_name: Annotated[
+        str | None,
+        Field(
+            description='Full name of the Actor',
+        ),
+    ] = None
+
+    actor_run_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_run_id',
+                'apify_actor_run_id',
+                'apify_act_run_id',
+            ),
+            description='ID of the Actor run',
+        ),
+    ] = None
+
+    actor_build_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_build_id',
+                'apify_actor_build_id',
+            ),
+            description='ID of the Actor build used in the run',
+        ),
+    ] = None
+
+    actor_build_number: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_build_number',
+                'apify_actor_build_number',
+            ),
+            description='Build number of the Actor build used in the run',
+        ),
+    ] = None
+
+    actor_build_tags: Annotated[
+        list[str] | None,
+        Field(
+            description='Build tags of the Actor build used in the run',
+        ),
+        BeforeValidator(_transform_to_list),
+    ] = None
+
+    actor_task_id: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_task_id',
+                'apify_actor_task_id',
+            ),
+            description='ID of the Actor task. Empty if Actor is run outside of any task, e.g. directly using the API',
+        ),
+    ] = None
+
+    actor_events_ws_url: Annotated[
+        str | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_events_websocket_url',
+                'apify_actor_events_ws_url',
+            ),
+            description='Websocket URL where Actor may listen for events from Actor platform',
+        ),
+    ] = None
+
+    api_base_url: Annotated[
+        str,
+        Field(
+            alias='apify_api_base_url',
+            description='Internal URL of the Apify API. May be used to interact with the platform programmatically',
+        ),
+    ] = 'https://api.apify.com'
+
+    api_public_base_url: Annotated[
+        str,
+        Field(
+            alias='apify_api_public_base_url',
+            description='Public URL of the Apify API. May be used to link to REST API resources',
+        ),
+    ] = 'https://api.apify.com'
+
+    dedicated_cpus: Annotated[
+        float | None,
+        Field(
+            alias='apify_dedicated_cpus',
+            description='Number of CPU cores reserved for the actor, based on allocated memory',
+        ),
+    ] = None
+
+    disable_outdated_warning: Annotated[
+        bool,
+        Field(
+            alias='apify_disable_outdated_warning',
+            description='Controls the display of outdated SDK version warnings',
+        ),
+        BeforeValidator(lambda val: val or False),
+    ] = False
+
+    fact: Annotated[str | None, Field(alias='apify_fact')] = None
+
+    input_key: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_input_key',
+                'apify_input_key',
+                'crawlee_input_key',
+            ),
+            description='Key of the record in the default key-value store that holds the Actor input',
+        ),
+    ] = 'INPUT'
+
+    input_secrets_private_key_file: Annotated[
+        str | None,
+        Field(
+            alias='apify_input_secrets_private_key_file',
+            description='Path to the secret key used to decrypt Secret inputs.',
+        ),
+    ] = None
+
+    input_secrets_private_key_passphrase: Annotated[
+        str | None,
+        Field(
+            alias='apify_input_secrets_private_key_passphrase',
+            description='Passphrase for the input secret key',
+        ),
+    ] = None
+
+    is_at_home: Annotated[
+        bool,
+        Field(
+            alias='apify_is_at_home',
+            description='True if the Actor is running on Apify servers',
+        ),
+    ] = False
+
+    latest_sdk_version: Annotated[
+        str | None,
+        Field(
+            alias='apify_sdk_latest_version',
+            description='Specifies the most recent release version of the Apify SDK for Javascript. Used for '
+            'checking for updates.',
+        ),
+        deprecated('SDK version checking is not supported for the Python SDK'),
+    ] = None
+
+    log_format: Annotated[
+        str | None,
+        Field(alias='apify_log_format'),
+        deprecated('Adjust the log format in code instead'),
+    ] = None
+
+    max_paid_dataset_items: Annotated[
+        int | None,
+        Field(
+            alias='actor_max_paid_dataset_items',
+            description='For paid-per-result Actors, the user-set limit on returned results. Do not exceed this limit',
+        ),
+        BeforeValidator(lambda val: val or None),
+    ] = None
+
+    max_total_charge_usd: Annotated[
+        float | None,
+        Field(
+            alias='actor_max_total_charge_usd',
+            description='For pay-per-event Actors, the user-set limit on total charges. Do not exceed this limit',
+        ),
+        BeforeValidator(lambda val: val or None),
+    ] = None
+
+    meta_origin: Annotated[
+        str | None,
+        Field(
+            alias='apify_meta_origin',
+            description='Specifies how an Actor run was started',
+        ),
+    ] = None
+
+    metamorph_after_sleep: Annotated[
+        timedelta_ms,
+        Field(
+            alias='apify_metamorph_after_sleep_millis',
+            description='How long the Actor needs to wait before exiting after triggering a metamorph',
+        ),
+    ] = timedelta(minutes=5)
+
+    proxy_hostname: Annotated[
+        str,
+        Field(
+            alias='apify_proxy_hostname',
+            description='Hostname of the Apify proxy',
+        ),
+    ] = 'proxy.apify.com'
+
+    proxy_password: Annotated[
+        str | None,
+        Field(
+            alias='apify_proxy_password',
+            description='Password to the Apify proxy',
+        ),
+    ] = None
+
+    proxy_port: Annotated[
+        int,
+        Field(
+            alias='apify_proxy_port',
+            description='Port to communicate with the Apify proxy',
+        ),
+    ] = 8000
+
+    proxy_status_url: Annotated[
+        str,
+        Field(
+            alias='apify_proxy_status_url',
+            description='URL for retrieving proxy status information',
+        ),
+    ] = 'http://proxy.apify.com'
+
+    started_at: Annotated[
+        datetime | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_started_at',
+                'apify_started_at',
+            ),
+            description='Date when the Actor was started',
+        ),
+    ] = None
+
+    timeout_at: Annotated[
+        datetime | None,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_timeout_at',
+                'apify_timeout_at',
+            ),
+            description='Date when the Actor will time out',
+        ),
+        BeforeValidator(lambda val: val if val != '' else None),  # We should accept empty environment variables as well
+    ] = None
+
+    standby_port: Annotated[
+        int,
+        Field(
+            alias='actor_standby_port',
+            description='TCP port for the Actor to start an HTTP server to receive messages in the Actor Standby mode',
+        ),
+        deprecated('Use `web_server_port` instead'),
+    ] = 4321
+
+    standby_url: Annotated[
+        str,
+        BeforeValidator(validate_http_url),
+        Field(
+            alias='actor_standby_url',
+            description='URL for accessing web servers of Actor runs in Standby mode',
+        ),
+    ] = 'http://localhost'
+
+    token: Annotated[
+        str | None,
+        Field(
+            alias='apify_token',
+            description='API token of the user who started the Actor',
+        ),
+    ] = None
+
+    user_id: Annotated[
+        str | None,
+        Field(
+            alias='apify_user_id',
+            description='ID of the user who started the Actor. May differ from the Actor owner',
+        ),
+    ] = None
+
+    web_server_port: Annotated[
+        int,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_web_server_port',
+                'apify_container_port',
+            ),
+            description='TCP port for the Actor to start an HTTP server on'
+            'This server can be used to receive external messages or expose monitoring and control interfaces',
+        ),
+    ] = 4321
+
+    web_server_url: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_web_server_url',
+                'apify_container_url',
+            ),
+            description='Unique public URL for accessing a specific Actor run web server from the outside world',
+        ),
+    ] = 'http://localhost:4321'
+
+    workflow_key: Annotated[
+        str | None,
+        Field(
+            alias='apify_workflow_key',
+            description='Identifier used for grouping related runs and API calls together',
+        ),
+    ] = None
+
+    @classmethod
+    def get_global_configuration(cls) -> Configuration:
+        """Retrieve the global instance of the configuration.
+
+        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
+        instead.
+        """
+        return cls()
+
+
+# Monkey-patch the base class so that it works with the extended configuration
+CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]

apify/_consts.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+import re
+from datetime import timedelta
+
+EVENT_LISTENERS_TIMEOUT = timedelta(seconds=5)
+
+BASE64_REGEXP = '[-A-Za-z0-9+/]*={0,3}'
+ENCRYPTED_INPUT_VALUE_PREFIX = 'ENCRYPTED_VALUE'
+ENCRYPTED_INPUT_VALUE_REGEXP = re.compile(f'^{ENCRYPTED_INPUT_VALUE_PREFIX}:({BASE64_REGEXP}):({BASE64_REGEXP})$')

apify/_crypto.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
 import base64
-import secrets
 from typing import Any
 
-from apify_shared.utils import ignore_docs
 from cryptography.exceptions import InvalidTag as InvalidTagException
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding, rsa
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
-from apify.consts import ENCRYPTED_INPUT_VALUE_REGEXP
+from apify_shared.utils import ignore_docs
+from crawlee._utils.crypto import crypto_random_object_id
+
+from apify._consts import ENCRYPTED_INPUT_VALUE_REGEXP
 
 ENCRYPTION_KEY_LENGTH = 32
 ENCRYPTION_IV_LENGTH = 16
@@ -25,11 +26,10 @@ def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     It returns the encrypted password and encrypted value in BASE64 format.
 
     Args:
-        value (str): The value which should be encrypted.
-        public_key (RSAPublicKey): Public key to use for encryption.
+        value: The value which should be encrypted.
+        public_key: Public key to use for encryption.
 
-    Returns:
-        disc: Encrypted password and value.
+    Returns: Encrypted password and value.
     """
     key_bytes = crypto_random_object_id(ENCRYPTION_KEY_LENGTH).encode('utf-8')
     initialized_vector_bytes = crypto_random_object_id(ENCRYPTION_IV_LENGTH).encode('utf-8')
@@ -37,8 +37,15 @@ def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
 
     password_bytes = key_bytes + initialized_vector_bytes
 
-    # NOTE: Auth Tag is appended to the end of the encrypted data, it has length of 16 bytes and ensures integrity of the data.
-    cipher = Cipher(algorithms.AES(key_bytes), modes.GCM(initialized_vector_bytes, min_tag_length=ENCRYPTION_AUTH_TAG_LENGTH))
+    # NOTE: Auth Tag is appended to the end of the encrypted data, it has length of 16 bytes and ensures integrity
+    # of the data.
+    cipher = Cipher(
+        algorithms.AES(key_bytes),
+        modes.GCM(
+            initialized_vector_bytes,
+            min_tag_length=ENCRYPTION_AUTH_TAG_LENGTH,
+        ),
+    )
     encryptor = cipher.encryptor()
     encrypted_value_bytes = encryptor.update(value_bytes) + encryptor.finalize()
     encrypted_password_bytes = public_key.encrypt(
@@ -65,12 +72,11 @@ def private_decrypt(
     """Decrypts the given encrypted value using the private key and password.
 
     Args:
-        encrypted_password (str): Password used to encrypt the private key encoded as base64 string.
-        encrypted_value (str): Encrypted value to decrypt as base64 string.
-        private_key (RSAPrivateKey): Private key to use for decryption.
+        encrypted_password: Password used to encrypt the private key encoded as base64 string.
+        encrypted_value: Encrypted value to decrypt as base64 string.
+        private_key: Private key to use for decryption.
 
-    Returns:
-        str: Decrypted value.
+    Returns: Decrypted value.
     """
     encrypted_password_bytes = base64.b64decode(encrypted_password.encode('utf-8'))
     encrypted_value_bytes = base64.b64decode(encrypted_value.encode('utf-8'))
@@ -95,7 +101,9 @@ def private_decrypt(
     initialization_vector_bytes = password_bytes[ENCRYPTION_KEY_LENGTH:]
 
     try:
-        cipher = Cipher(algorithms.AES(encryption_key_bytes), modes.GCM(initialization_vector_bytes, authentication_tag_bytes))
+        cipher = Cipher(
+            algorithms.AES(encryption_key_bytes), modes.GCM(initialization_vector_bytes, authentication_tag_bytes)
+        )
         decryptor = cipher.decryptor()
         decipher_bytes = decryptor.update(encrypted_data_bytes) + decryptor.finalize()
     except InvalidTagException as exc:
@@ -106,6 +114,7 @@ def private_decrypt(
     return decipher_bytes.decode('utf-8')
 
 
+@ignore_docs
 def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey:
     private_key = serialization.load_pem_private_key(
         base64.b64decode(private_key_file_base64.encode('utf-8')),
@@ -125,27 +134,22 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey:
     return public_key
 
 
-def crypto_random_object_id(length: int = 17) -> str:
-    """Python reimplementation of cryptoRandomObjectId from `@apify/utilities`."""
-    chars = 'abcdefghijklmnopqrstuvwxyzABCEDFGHIJKLMNOPQRSTUVWXYZ0123456789'
-    return ''.join(secrets.choice(chars) for _ in range(length))
-
-
-def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input: Any) -> Any:  # noqa: A002
+@ignore_docs
+def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any:
     """Decrypt input secrets."""
-    if not isinstance(input, dict):
-        return input
+    if not isinstance(input_data, dict):
+        return input_data
 
-    for key, value in input.items():
+    for key, value in input_data.items():
         if isinstance(value, str):
             match = ENCRYPTED_INPUT_VALUE_REGEXP.fullmatch(value)
             if match:
                 encrypted_password = match.group(1)
                 encrypted_value = match.group(2)
-                input[key] = private_decrypt(
+                input_data[key] = private_decrypt(
                     encrypted_password,
                     encrypted_value,
                     private_key=private_key,
                 )
 
-    return input
+    return input_data

apify/_models.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from typing import Annotated
+
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
+
+from apify_shared.consts import ActorJobStatus, MetaOrigin, WebhookEventType
+from crawlee._utils.models import timedelta_ms
+from crawlee._utils.urls import validate_http_url
+
+from apify._utils import docs_group
+
+
+@docs_group('Data structures')
+class Webhook(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    event_types: Annotated[
+        list[WebhookEventType],
+        Field(description='Event types that should trigger the webhook'),
+    ]
+    request_url: Annotated[
+        str,
+        Field(description='URL that the webhook should call'),
+        BeforeValidator(validate_http_url),
+    ]
+    payload_template: Annotated[
+        str | None,
+        Field(description='Template for the payload sent by the webook'),
+    ] = None
+
+
+@docs_group('Data structures')
+class ActorRunMeta(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    origin: Annotated[MetaOrigin, Field()]
+
+
+@docs_group('Data structures')
+class ActorRunStats(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    input_body_len: Annotated[int, Field(alias='inputBodyLen')]
+    restart_count: Annotated[int, Field(alias='restartCount')]
+    resurrect_count: Annotated[int, Field(alias='resurrectCount')]
+    mem_avg_bytes: Annotated[float | None, Field(alias='memAvgBytes')] = None
+    mem_max_bytes: Annotated[int | None, Field(alias='memMaxBytes')] = None
+    mem_current_bytes: Annotated[int | None, Field(alias='memCurrentBytes')] = None
+    cpu_avg_usage: Annotated[float | None, Field(alias='cpuAvgUsage')] = None
+    cpu_max_usage: Annotated[float | None, Field(alias='cpuMaxUsage')] = None
+    cpu_current_usage: Annotated[float | None, Field(alias='cpuCurrentUsage')] = None
+    net_rx_bytes: Annotated[int | None, Field(alias='netRxBytes')] = None
+    net_tx_bytes: Annotated[int | None, Field(alias='netTxBytes')] = None
+    duration: Annotated[timedelta_ms | None, Field(alias='durationMillis')] = None
+    run_time: Annotated[timedelta | None, Field(alias='runTimeSecs')] = None
+    metamorph: Annotated[int | None, Field(alias='metamorph')] = None
+    compute_units: Annotated[float, Field(alias='computeUnits')]
+
+
+@docs_group('Data structures')
+class ActorRunOptions(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    build: str
+    timeout: Annotated[timedelta, Field(alias='timeoutSecs')]
+    memory_mbytes: Annotated[int, Field(alias='memoryMbytes')]
+    disk_mbytes: Annotated[int, Field(alias='diskMbytes')]
+
+
+@docs_group('Data structures')
+class ActorRunUsage(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    actor_compute_units: Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')] = None
+    dataset_reads: Annotated[float | None, Field(alias='DATASET_READS')] = None
+    dataset_writes: Annotated[float | None, Field(alias='DATASET_WRITES')] = None
+    key_value_store_reads: Annotated[float | None, Field(alias='KEY_VALUE_STORE_READS')] = None
+    key_value_store_writes: Annotated[float | None, Field(alias='KEY_VALUE_STORE_WRITES')] = None
+    key_value_store_lists: Annotated[float | None, Field(alias='KEY_VALUE_STORE_LISTS')] = None
+    request_queue_reads: Annotated[float | None, Field(alias='REQUEST_QUEUE_READS')] = None
+    request_queue_writes: Annotated[float | None, Field(alias='REQUEST_QUEUE_WRITES')] = None
+    data_transfer_internal_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')] = None
+    data_transfer_external_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')] = None
+    proxy_residential_transfer_gbytes: Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')] = None
+    proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
+
+
+@docs_group('Data structures')
+class ActorRun(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    id: Annotated[str, Field(alias='id')]
+    act_id: Annotated[str, Field(alias='actId')]
+    user_id: Annotated[str, Field(alias='userId')]
+    actor_task_id: Annotated[str | None, Field(alias='actorTaskId')] = None
+    started_at: Annotated[datetime, Field(alias='startedAt')]
+    finished_at: Annotated[datetime | None, Field(alias='finishedAt')] = None
+    status: Annotated[ActorJobStatus, Field(alias='status')]
+    status_message: Annotated[str | None, Field(alias='statusMessage')] = None
+    is_status_message_terminal: Annotated[bool | None, Field(alias='isStatusMessageTerminal')] = None
+    meta: Annotated[ActorRunMeta, Field(alias='meta')]
+    stats: Annotated[ActorRunStats, Field(alias='stats')]
+    options: Annotated[ActorRunOptions, Field(alias='options')]
+    build_id: Annotated[str, Field(alias='buildId')]
+    exit_code: Annotated[int | None, Field(alias='exitCode')] = None
+    default_key_value_store_id: Annotated[str, Field(alias='defaultKeyValueStoreId')]
+    default_dataset_id: Annotated[str, Field(alias='defaultDatasetId')]
+    default_request_queue_id: Annotated[str, Field(alias='defaultRequestQueueId')]
+    build_number: Annotated[str | None, Field(alias='buildNumber')] = None
+    container_url: Annotated[str, Field(alias='containerUrl')]
+    is_container_server_ready: Annotated[bool | None, Field(alias='isContainerServerReady')] = None
+    git_branch_name: Annotated[str | None, Field(alias='gitBranchName')] = None
+    usage: Annotated[ActorRunUsage | None, Field(alias='usage')] = None
+    usage_total_usd: Annotated[float | None, Field(alias='usageTotalUsd')] = None
+    usage_usd: Annotated[ActorRunUsage | None, Field(alias='usageUsd')] = None