apify 2.0.0b6__tar.gz → 2.0.0b7__tar.gz

{apify-2.0.0b6 → apify-2.0.0b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 2.0.0b6
+Version: 2.0.0b7
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -18,21 +18,15 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
-Requires-Dist: aiofiles (>=22.1.0,<23.0.0)
-Requires-Dist: aioshutil (>=1.0,<2.0)
-Requires-Dist: apify-client (>=1.7.1,<2.0.0)
-Requires-Dist: apify-shared (>=1.1.2,<2.0.0)
-Requires-Dist: colorama (>=0.4.6,<0.5.0)
-Requires-Dist: crawlee (>=0.3.0,<0.4.0)
-Requires-Dist: cryptography (>=42.0.4,<43.0.0)
-Requires-Dist: httpx (>=0.27.0,<0.28.0)
-Requires-Dist: lazy-object-proxy (>=1.10.0,<2.0.0)
-Requires-Dist: psutil (>=6.0.0,<7.0.0)
-Requires-Dist: pyee (>=11.0.0,<12.0.0)
-Requires-Dist: scrapy (>=2.11.0,<3.0.0) ; extra == "scrapy"
-Requires-Dist: sortedcollections (>=2.0.0,<3.0.0)
-Requires-Dist: typing-extensions (>=4.1.0,<5.0.0)
-Requires-Dist: websockets (>=10.1,<11.0)
+Requires-Dist: apify-client (>=1.7.1)
+Requires-Dist: apify-shared (>=1.1.2)
+Requires-Dist: crawlee (>=0.3.0)
+Requires-Dist: cryptography (>=42.0.0)
+Requires-Dist: httpx (>=0.27.0)
+Requires-Dist: lazy-object-proxy (>=1.10.0)
+Requires-Dist: scrapy (>=2.11.0) ; extra == "scrapy"
+Requires-Dist: typing-extensions (>=4.1.0)
+Requires-Dist: websockets (>=10.0)
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
 Project-URL: Documentation, https://docs.apify.com/sdk/python/

{apify-2.0.0b6 → apify-2.0.0b7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify"
-version = "2.0.0b6"
+version = "2.0.0b7"
 description = "Apify SDK for Python"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 license = "Apache-2.0"
@@ -46,41 +46,32 @@ keywords = [
 # https://github.com/apify/apify-sdk-python/pull/154.
 [tool.poetry.dependencies]
 python = "^3.9"
-aiofiles = "^22.1.0"
-aioshutil = "^1.0"
-apify-client = "^1.7.1"
-apify-shared = "^1.1.2"
-colorama = "^0.4.6"
-crawlee = "^0.3.0"
-cryptography = "^42.0.4"
-httpx = "^0.27.0"
-lazy-object-proxy = "^1.10.0"
-psutil = "^6.0.0"
-pyee = "^11.0.0"
-scrapy = { version = "^2.11.0", optional = true }
-sortedcollections = "^2.0.0"
-typing-extensions = "^4.1.0"
-websockets = "^10.1"
+apify-client = ">=1.7.1"
+apify-shared = ">=1.1.2"
+crawlee = ">=0.3.0"
+cryptography = ">=42.0.0"
+httpx = ">=0.27.0"
+lazy-object-proxy = ">=1.10.0"
+scrapy = { version = ">=2.11.0", optional = true }
+typing-extensions = ">=4.1.0"
+websockets = ">=10.0"
 
 [tool.poetry.group.dev.dependencies]
-build = "^1.2.0"
-filelock = "^3.15.0"
-mypy = "^1.11.0"
-pre-commit = "^3.8.0"
-pydoc-markdown = "^4.8.0"
-pytest = "^8.3.0"
-pytest-asyncio = "^0.24.0"
-pytest-cov = "^5.0.0"
-pytest-only = "^2.1.0"
-pytest-timeout = "^2.3.0"
-pytest-xdist = "^3.6.0"
-respx = "^0.21.0"
-ruff = "^0.6.0"
-setuptools = "^74.0.0"              # setuptools are used by pytest but not explicitly required
-twine = "^5.1.0"
-types-aiofiles = "^24.1.0.20240626"
-types-colorama = "^0.4.15.20240311"
-types-psutil = "^6.0.0.20240621"
+build = "~1.2.0"
+filelock = "~3.15.0"
+griffe = "~1.2.0"
+mypy = "~1.11.0"
+pre-commit = "~3.8.0"
+pydoc-markdown = "~4.8.0"
+pytest = "~8.3.0"
+pytest-asyncio = "~0.24.0"
+pytest-cov = "~5.0.0"
+pytest-only = "~2.1.0"
+pytest-timeout = "~2.3.0"
+pytest-xdist = "~3.6.0"
+respx = "~0.21.0"
+ruff = "~0.6.0"
+setuptools = "~74.0.0"     # setuptools are used by pytest but not explicitly required
 
 [tool.poetry.extras]
 scrapy = ["scrapy"]
@@ -191,7 +182,7 @@ warn_unused_ignores = true
 exclude = []
 
 [[tool.mypy.overrides]]
-module = ['scrapy', 'scrapy.*', 'sortedcollections', 'lazy_object_proxy']
+module = ['scrapy', 'scrapy.*', 'lazy_object_proxy']
 ignore_missing_imports = true
 
 [tool.coverage.report]

{apify-2.0.0b6 → apify-2.0.0b7}/src/apify/__init__.py

@@ -1,11 +1,22 @@
 from importlib import metadata
 
+from apify_shared.consts import WebhookEventType
 from crawlee.events._types import Event
 
 from apify._actor import Actor
 from apify._configuration import Configuration
+from apify._models import Webhook
 from apify._proxy_configuration import ProxyConfiguration, ProxyInfo
 
 __version__ = metadata.version('apify')
 
-__all__ = ['Actor', 'Event', 'Configuration', 'ProxyConfiguration', 'ProxyInfo', '__version__']
+__all__ = [
+    'Actor',
+    'Event',
+    'Configuration',
+    'ProxyConfiguration',
+    'ProxyInfo',
+    'Webhook',
+    'WebhookEventType',
+    '__version__',
+]

{apify-2.0.0b6 → apify-2.0.0b7}/src/apify/_actor.py

@@ -11,7 +11,7 @@ from pydantic import AliasChoices
 from typing_extensions import Self
 
 from apify_client import ApifyClientAsync
-from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars, WebhookEventType
+from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
 from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
 from crawlee import service_container
 from crawlee.events._types import Event, EventPersistStateData
@@ -19,11 +19,12 @@ from crawlee.events._types import Event, EventPersistStateData
 from apify._configuration import Configuration
 from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
-from apify._log import logger
+from apify._models import ActorRun
 from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
 from apify._utils import get_system_info, is_running_in_ipython
 from apify.apify_storage_client import ApifyStorageClient
+from apify.log import logger
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -32,6 +33,8 @@ if TYPE_CHECKING:
 
     from crawlee.proxy_configuration import _NewUrlFunction
 
+    from apify._models import Webhook
+
 
 MainReturnType = TypeVar('MainReturnType')
 
@@ -533,8 +536,8 @@ class _ActorType:
         memory_mbytes: int | None = None,
         timeout: timedelta | None = None,
         wait_for_finish: int | None = None,
-        webhooks: list[dict] | None = None,
-    ) -> dict:
+        webhooks: list[Webhook] | None = None,
+    ) -> ActorRun:
         """Run an Actor on the Apify platform.
 
         Unlike `Actor.call`, this method just starts the run without waiting for finish.
@@ -555,10 +558,6 @@ class _ActorType:
             webhooks: Optional ad-hoc webhooks (https://docs.apify.com/webhooks/ad-hoc-webhooks) associated with
                 the Actor run which can be used to receive a notification, e.g. when the Actor finished or failed.
                 If you already have a webhook set up for the Actor or task, you do not have to add it again here.
-                Each webhook is represented by a dictionary containing these items:
-                    * `event_types`: list of `WebhookEventType` values which trigger the webhook
-                    * `request_url`: URL to which to send the webhook HTTP request
-                    * `payload_template` (optional): Optional template for the request payload
 
         Returns:
             Info about the started Actor run
@@ -567,16 +566,25 @@ class _ActorType:
 
         client = self.new_client(token=token) if token else self._apify_client
 
-        return await client.actor(actor_id).start(
+        if webhooks:
+            serialized_webhooks = [
+                hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
+            ]
+        else:
+            serialized_webhooks = None
+
+        api_result = await client.actor(actor_id).start(
             run_input=run_input,
             content_type=content_type,
             build=build,
             memory_mbytes=memory_mbytes,
             timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
             wait_for_finish=wait_for_finish,
-            webhooks=webhooks,
+            webhooks=serialized_webhooks,
         )
 
+        return ActorRun.model_validate(api_result)
+
     async def abort(
         self,
         run_id: str,
@@ -584,7 +592,7 @@ class _ActorType:
         token: str | None = None,
         status_message: str | None = None,
         gracefully: bool | None = None,
-    ) -> dict:
+    ) -> ActorRun:
         """Abort given Actor run on the Apify platform using the current user account.
 
         The user account is determined by the `APIFY_TOKEN` environment variable.
@@ -607,7 +615,9 @@ class _ActorType:
         if status_message:
             await client.run(run_id).update(status_message=status_message)
 
-        return await client.run(run_id).abort(gracefully=gracefully)
+        api_result = await client.run(run_id).abort(gracefully=gracefully)
+
+        return ActorRun.model_validate(api_result)
 
     async def call(
         self,
@@ -619,9 +629,9 @@ class _ActorType:
         build: str | None = None,
         memory_mbytes: int | None = None,
         timeout: timedelta | None = None,
-        webhooks: list[dict] | None = None,
+        webhooks: list[Webhook] | None = None,
         wait: timedelta | None = None,
-    ) -> dict | None:
+    ) -> ActorRun | None:
         """Start an Actor on the Apify Platform and wait for it to finish before returning.
 
         It waits indefinitely, unless the wait argument is provided.
@@ -650,16 +660,25 @@ class _ActorType:
 
         client = self.new_client(token=token) if token else self._apify_client
 
-        return await client.actor(actor_id).call(
+        if webhooks:
+            serialized_webhooks = [
+                hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
+            ]
+        else:
+            serialized_webhooks = None
+
+        api_result = await client.actor(actor_id).call(
             run_input=run_input,
             content_type=content_type,
             build=build,
             memory_mbytes=memory_mbytes,
             timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
-            webhooks=webhooks,
+            webhooks=serialized_webhooks,
             wait_secs=int(wait.total_seconds()) if wait is not None else None,
         )
 
+        return ActorRun.model_validate(api_result)
+
     async def call_task(
         self,
         task_id: str,
@@ -668,10 +687,10 @@ class _ActorType:
         build: str | None = None,
         memory_mbytes: int | None = None,
         timeout: timedelta | None = None,
-        webhooks: list[dict] | None = None,
+        webhooks: list[Webhook] | None = None,
         wait: timedelta | None = None,
         token: str | None = None,
-    ) -> dict | None:
+    ) -> ActorRun | None:
         """Start an Actor task on the Apify Platform and wait for it to finish before returning.
 
         It waits indefinitely, unless the wait argument is provided.
@@ -703,15 +722,24 @@ class _ActorType:
 
         client = self.new_client(token=token) if token else self._apify_client
 
-        return await client.task(task_id).call(
+        if webhooks:
+            serialized_webhooks = [
+                hook.model_dump(by_alias=True, exclude_unset=True, exclude_defaults=True) for hook in webhooks
+            ]
+        else:
+            serialized_webhooks = None
+
+        api_result = await client.task(task_id).call(
             task_input=task_input,
             build=build,
             memory_mbytes=memory_mbytes,
             timeout_secs=int(timeout.total_seconds()) if timeout is not None else None,
-            webhooks=webhooks,
+            webhooks=serialized_webhooks,
             wait_secs=int(wait.total_seconds()) if wait is not None else None,
         )
 
+        return ActorRun.model_validate(api_result)
+
     async def metamorph(
         self,
         target_actor_id: str,
@@ -796,14 +824,12 @@ class _ActorType:
 
     async def add_webhook(
         self,
+        webhook: Webhook,
         *,
-        event_types: list[WebhookEventType],
-        request_url: str,
-        payload_template: str | None = None,
         ignore_ssl_errors: bool | None = None,
         do_not_retry: bool | None = None,
         idempotency_key: str | None = None,
-    ) -> dict | None:
+    ) -> None:
         """Create an ad-hoc webhook for the current Actor run.
 
         This webhook lets you receive a notification when the Actor run finished or failed.
@@ -814,9 +840,7 @@ class _ActorType:
         For more information about Apify Actor webhooks, please see the [documentation](https://docs.apify.com/webhooks).
 
         Args:
-            event_types: List of event types that should trigger the webhook. At least one is required.
-            request_url: URL that will be invoked once the webhook is triggered.
-            payload_template: Specification of the payload that will be sent to request_url
+            webhook: The webhook to be added
             ignore_ssl_errors: Whether the webhook should ignore SSL errors returned by request_url
             do_not_retry: Whether the webhook should retry sending the payload to request_url upon failure.
             idempotency_key: A unique identifier of a webhook. You can use it to ensure that you won't create
@@ -829,17 +853,17 @@ class _ActorType:
 
         if not self.is_at_home():
             self.log.error('Actor.add_webhook() is only supported when running on the Apify platform.')
-            return None
+            return
 
         # If is_at_home() is True, config.actor_run_id is always set
         if not self._configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        return await self._apify_client.webhooks().create(
+        await self._apify_client.webhooks().create(
             actor_run_id=self._configuration.actor_run_id,
-            event_types=event_types,
-            request_url=request_url,
-            payload_template=payload_template,
+            event_types=webhook.event_types,
+            request_url=webhook.request_url,
+            payload_template=webhook.payload_template,
             ignore_ssl_errors=ignore_ssl_errors,
             do_not_retry=do_not_retry,
             idempotency_key=idempotency_key,
@@ -850,7 +874,7 @@ class _ActorType:
         status_message: str,
         *,
         is_terminal: bool | None = None,
-    ) -> dict | None:
+    ) -> ActorRun | None:
         """Set the status message for the current Actor run.
 
         Args:
@@ -871,10 +895,12 @@ class _ActorType:
         if not self._configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        return await self._apify_client.run(self._configuration.actor_run_id).update(
+        api_result = await self._apify_client.run(self._configuration.actor_run_id).update(
             status_message=status_message, is_status_message_terminal=is_terminal
         )
 
+        return ActorRun.model_validate(api_result)
+
     async def create_proxy_configuration(
         self,
         *,

apify-2.0.0b7/src/apify/_models.py

@@ -0,0 +1,110 @@
+# ruff: noqa: TCH001 TCH002 TCH003 (Pydantic)
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from typing import Annotated
+
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
+
+from apify_shared.consts import ActorJobStatus, MetaOrigin, WebhookEventType
+from crawlee._utils.models import timedelta_ms
+from crawlee._utils.urls import validate_http_url
+
+
+class Webhook(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    event_types: Annotated[
+        list[WebhookEventType],
+        Field(description='Event types that should trigger the webhook'),
+    ]
+    request_url: Annotated[
+        str,
+        Field(description='URL that the webhook should call'),
+        BeforeValidator(validate_http_url),
+    ]
+    payload_template: Annotated[
+        str | None,
+        Field(description='Template for the payload sent by the webook'),
+    ] = None
+
+
+class ActorRunMeta(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    origin: Annotated[MetaOrigin, Field()]
+
+
+class ActorRunStats(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    input_body_len: Annotated[int, Field(alias='inputBodyLen')]
+    restart_count: Annotated[int, Field(alias='restartCount')]
+    resurrect_count: Annotated[int, Field(alias='resurrectCount')]
+    mem_avg_bytes: Annotated[float | None, Field(alias='memAvgBytes')] = None
+    mem_max_bytes: Annotated[int | None, Field(alias='memMaxBytes')] = None
+    mem_current_bytes: Annotated[int | None, Field(alias='memCurrentBytes')] = None
+    cpu_avg_usage: Annotated[float | None, Field(alias='cpuAvgUsage')] = None
+    cpu_max_usage: Annotated[float | None, Field(alias='cpuMaxUsage')] = None
+    cpu_current_usage: Annotated[float | None, Field(alias='cpuCurrentUsage')] = None
+    net_rx_bytes: Annotated[int | None, Field(alias='netRxBytes')] = None
+    net_tx_bytes: Annotated[int | None, Field(alias='netTxBytes')] = None
+    duration: Annotated[timedelta_ms | None, Field(alias='durationMillis')] = None
+    run_time: Annotated[timedelta | None, Field(alias='runTimeSecs')] = None
+    metamorph: Annotated[int | None, Field(alias='metamorph')] = None
+    compute_units: Annotated[float, Field(alias='computeUnits')]
+
+
+class ActorRunOptions(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    build: str
+    timeout: Annotated[timedelta, Field(alias='timeoutSecs')]
+    memory_mbytes: Annotated[int, Field(alias='memoryMbytes')]
+    disk_mbytes: Annotated[int, Field(alias='diskMbytes')]
+
+
+class ActorRunUsage(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    actor_compute_units: Annotated[float | None, Field(alias='ACTOR_COMPUTE_UNITS')] = None
+    dataset_reads: Annotated[float | None, Field(alias='DATASET_READS')] = None
+    dataset_writes: Annotated[float | None, Field(alias='DATASET_WRITES')] = None
+    key_value_store_reads: Annotated[float | None, Field(alias='KEY_VALUE_STORE_READS')] = None
+    key_value_store_writes: Annotated[float | None, Field(alias='KEY_VALUE_STORE_WRITES')] = None
+    key_value_store_lists: Annotated[float | None, Field(alias='KEY_VALUE_STORE_LISTS')] = None
+    request_queue_reads: Annotated[float | None, Field(alias='REQUEST_QUEUE_READS')] = None
+    request_queue_writes: Annotated[float | None, Field(alias='REQUEST_QUEUE_WRITES')] = None
+    data_transfer_internal_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_INTERNAL_GBYTES')] = None
+    data_transfer_external_gbytes: Annotated[float | None, Field(alias='DATA_TRANSFER_EXTERNAL_GBYTES')] = None
+    proxy_residential_transfer_gbytes: Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')] = None
+    proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
+
+
+class ActorRun(BaseModel):
+    __model_config__ = ConfigDict(populate_by_name=True)
+
+    id: Annotated[str, Field(alias='id')]
+    act_id: Annotated[str, Field(alias='actId')]
+    user_id: Annotated[str, Field(alias='userId')]
+    actor_task_id: Annotated[str | None, Field(alias='actorTaskId')] = None
+    started_at: Annotated[datetime, Field(alias='startedAt')]
+    finished_at: Annotated[datetime | None, Field(alias='finishedAt')] = None
+    status: Annotated[ActorJobStatus, Field(alias='status')]
+    status_message: Annotated[str | None, Field(alias='statusMessage')] = None
+    is_status_message_terminal: Annotated[bool | None, Field(alias='isStatusMessageTerminal')] = None
+    meta: Annotated[ActorRunMeta, Field(alias='meta')]
+    stats: Annotated[ActorRunStats, Field(alias='stats')]
+    options: Annotated[ActorRunOptions, Field(alias='options')]
+    build_id: Annotated[str, Field(alias='buildId')]
+    exit_code: Annotated[int | None, Field(alias='exitCode')] = None
+    default_key_value_store_id: Annotated[str, Field(alias='defaultKeyValueStoreId')]
+    default_dataset_id: Annotated[str, Field(alias='defaultDatasetId')]
+    default_request_queue_id: Annotated[str, Field(alias='defaultRequestQueueId')]
+    build_number: Annotated[str | None, Field(alias='buildNumber')] = None
+    container_url: Annotated[str, Field(alias='containerUrl')]
+    is_container_server_ready: Annotated[bool | None, Field(alias='isContainerServerReady')] = None
+    git_branch_name: Annotated[str | None, Field(alias='gitBranchName')] = None
+    usage: Annotated[ActorRunUsage | None, Field(alias='usage')] = None
+    usage_total_usd: Annotated[float | None, Field(alias='usageTotalUsd')] = None
+    usage_usd: Annotated[ActorRunUsage | None, Field(alias='usageUsd')] = None

{apify-2.0.0b6 → apify-2.0.0b7}/src/apify/_platform_event_manager.py

@@ -20,7 +20,7 @@ from crawlee.events._types import (
     EventSystemInfoData,
 )
 
-from apify._log import logger
+from apify.log import logger
 
 if TYPE_CHECKING:
     from types import TracebackType

{apify-2.0.0b6 → apify-2.0.0b7}/src/apify/_proxy_configuration.py

@@ -16,7 +16,7 @@ from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
 from crawlee.proxy_configuration import _NewUrlFunction
 
 from apify._configuration import Configuration
-from apify._log import logger
+from apify.log import logger
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync

apify-2.0.0b6/src/apify/_log.py → apify-2.0.0b7/src/apify/log.py

@@ -11,5 +11,5 @@ logger_name = __name__.split('.')[0]
 logger = logging.getLogger(logger_name)
 
 
-class ActorLogFormatter(CrawleeLogFormatter):  # Inherited from parent class
+class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
     pass