apify 2.7.0__tar.gz → 2.7.0b2__tar.gz

{apify-2.7.0 → apify-2.7.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 2.7.0
+Version: 2.7.0b2
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-2.7.0 → apify-2.7.0b2}/docs/03_concepts/10_logging.mdx

@@ -7,6 +7,8 @@ import CodeBlock from '@theme/CodeBlock';
 
 import LogConfigExample from '!!raw-loader!./code/10_log_config.py';
 import LoggerUsageExample from '!!raw-loader!./code/10_logger_usage.py';
+import RedirectLog from '!!raw-loader!./code/10_redirect_log.py';
+import RedirectLogExistingRun from '!!raw-loader!./code/10_redirect_log_existing_run.py';
 
 The Apify SDK is logging useful information through the [`logging`](https://docs.python.org/3/library/logging.html) module from Python's standard library, into the logger with the name `apify`.
 
@@ -82,3 +84,31 @@ Result:
 </pre>
 
 <!-- markdownlint-enable no-inline-html -->
+
+## Redirect logs from other Actor runs
+
+In some situations, one Actor is going to start one or more other Actors and wait for them to finish and produce some results. In such cases, you might want to redirect the logs and status messages of the started Actors runs back to the parent Actor run, so that you can see the progress of the started Actors' runs in the parent Actor's logs. This guide will show possibilities on how to do it.
+
+### Redirecting logs from Actor.call
+
+Typical use case for log redirection is to call another Actor using the [`Actor.call`](../../reference/class/Actor#call) method. This method has an optional `logger` argument, which is by default set to the `default` literal. This means that the logs of the called Actor will be automatically redirected to the parent Actor's logs with default formatting and filtering. If you set the `logger` argument to `None`, then no log redirection happens. The third option is to pass your own `Logger` instance with the possibility to define your own formatter, filter, and handler. Below you can see those three possible ways of log redirection when starting another Actor run through [`Actor.call`](../../reference/class/Actor#call).
+
+<CodeBlock className="language-python">
+    {RedirectLog}
+</CodeBlock>
+
+Each default redirect logger log entry will have a specific format. After the timestamp, it will contain cyan colored text that will contain the redirect information - the other actor's name and the run ID. The rest of the log message will be printed in the same manner as the parent Actor's logger is configured.
+
+The log redirection can be deep, meaning that if the other actor also starts another actor and is redirecting logs from it, then in the top-level Actor, you can see it as well. See the following example screenshot of the Apify log console when one actor recursively starts itself (there are 2 levels of recursion in the example).
+
+![Console with redirected logs](/img/guides/redirected_logs_example.webp 'Example of console with redirected logs from recursively started actor.')
+
+### Redirecting logs from already running Actor run
+
+In some cases, you might want to connect to an already running Actor run and redirect its logs to your current Actor run. This can be done using the [ApifyClient](../../reference/class/Actor#apify_client) and getting the streamed log from a specific Actor run. You can then use it as a context manager, and the log redirection will be active in the context, or you can control the log redirection manually by explicitly calling `start` and `stop` methods.
+
+You can further decide whether you want to redirect just new logs of the ongoing Actor run, or if you also want to redirect historical logs from that Actor's run, so all logs it has produced since it was started. Both options are shown in the example code below.
+
+<CodeBlock className="language-python">
+    {RedirectLogExistingRun}
+</CodeBlock>

apify-2.7.0b2/docs/03_concepts/code/10_redirect_log.py

@@ -0,0 +1,15 @@
+import logging
+
+from apify import Actor
+
+
+async def main() -> None:
+    async with Actor:
+        # Default redirect logger
+        await Actor.call(actor_id='some_actor_id')
+        # No redirect logger
+        await Actor.call(actor_id='some_actor_id', logger=None)
+        # Custom redirect logger
+        await Actor.call(
+            actor_id='some_actor_id', logger=logging.getLogger('custom_logger')
+        )

apify-2.7.0b2/docs/03_concepts/code/10_redirect_log_existing_run.py

@@ -0,0 +1,24 @@
+import asyncio
+
+from apify import Actor
+
+
+async def main() -> None:
+    async with Actor:
+        # Lifecycle of redirected logs is handled by the context manager.
+        async with await Actor.apify_client.run('some_actor_id').get_streamed_log(
+            # Redirect all logs from the start of that run, even the logs from past.
+            from_start=True
+        ):
+            await asyncio.sleep(5)
+            # Logging will stop out of context
+
+        # Lifecycle of redirected logs can be handled manually.
+        streamed_log = await Actor.apify_client.run('some_id').get_streamed_log(
+            # Do not redirect historical logs from this actor run.
+            # Redirect only new logs from now on.
+            from_start=False
+        )
+        streamed_log.start()
+        await asyncio.sleep(5)
+        await streamed_log.stop()

{apify-2.7.0 → apify-2.7.0b2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "2.7.0"
+version = "2.7.0b2"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -64,11 +64,11 @@ scrapy = ["scrapy>=2.11.0"]
 dev = [
     "build~=1.2.0",
     "dycw-pytest-only>=2.1.1",
-    "griffe~=1.7.0",
-    "mypy~=1.16.0",
+    "griffe~=1.9.0",
+    "mypy~=1.17.0",
     "pre-commit~=4.2.0",
     "pydoc-markdown~=4.8.0",
-    "pytest-asyncio~=1.0.0",
+    "pytest-asyncio~=1.1.0",
     "pytest-cov~=6.2.0",
     "pytest-timeout>=2.4.0",
     "pytest-xdist~=3.8.0",

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_actor.py

@@ -13,7 +13,7 @@ from pydantic import AliasChoices
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
-from apify_shared.utils import ignore_docs, maybe_extract_enum_member_value
+from apify_shared.utils import maybe_extract_enum_member_value
 from crawlee import service_locator
 from crawlee.events import (
     Event,
@@ -54,9 +54,46 @@ MainReturnType = TypeVar('MainReturnType')
 
 
 @docs_name('Actor')
-@docs_group('Classes')
+@docs_group('Actor')
 class _ActorType:
-    """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
+    """The core class for building Actors on the Apify platform.
+
+    Actors are serverless programs running in the cloud that can perform anything from simple actions
+    (such as filling out a web form or sending an email) to complex operations (such as crawling an
+    entire website or removing duplicates from a large dataset). They are packaged as Docker containers
+    which accept well-defined JSON input, perform an action, and optionally produce well-defined output.
+
+    ### References
+
+    - Apify platform documentation: https://docs.apify.com/platform/actors
+    - Actor whitepaper: https://whitepaper.actor/
+
+    ### Usage
+
+    ```python
+    import asyncio
+
+    import httpx
+    from apify import Actor
+    from bs4 import BeautifulSoup
+
+
+    async def main() -> None:
+        async with Actor:
+            actor_input = await Actor.get_input()
+            async with httpx.AsyncClient() as client:
+                response = await client.get(actor_input['url'])
+            soup = BeautifulSoup(response.content, 'html.parser')
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string if soup.title else None,
+            }
+            await Actor.push_data(data)
+
+    if __name__ == '__main__':
+        asyncio.run(main())
+    ```
+    """
 
     _is_rebooting = False
     _is_any_instance_initialized = False
@@ -108,7 +145,6 @@ class _ActorType:
 
         self._is_initialized = False
 
-    @ignore_docs
     async def __aenter__(self) -> Self:
         """Initialize the Actor.
 
@@ -120,7 +156,6 @@ class _ActorType:
         await self.init()
         return self
 
-    @ignore_docs
     async def __aexit__(
         self,
         _exc_type: type[BaseException] | None,

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_charging.py

@@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Protocol
 
 from pydantic import TypeAdapter
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.context import ensure_context
 
 from apify._models import ActorRun, PricingModel
@@ -26,9 +25,18 @@ if TYPE_CHECKING:
 run_validator = TypeAdapter[ActorRun | None](ActorRun | None)
 
 
-@docs_group('Interfaces')
+@docs_group('Charging')
 class ChargingManager(Protocol):
-    """Provides fine-grained access to pay-per-event functionality."""
+    """Provides fine-grained access to pay-per-event functionality.
+
+    The ChargingManager allows you to charge for specific events in your Actor when using
+    the pay-per-event pricing model. This enables precise cost control and transparent
+    billing for different operations within your Actor.
+
+    ### References
+
+    - Apify platform documentation: https://docs.apify.com/platform/actors/publishing/monetize
+    """
 
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
@@ -57,7 +65,7 @@ class ChargingManager(Protocol):
         """
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass(frozen=True)
 class ChargeResult:
     """Result of the `ChargingManager.charge` method."""
@@ -72,7 +80,7 @@ class ChargeResult:
     """How many events of each known type can still be charged within the limit."""
 
 
-@docs_group('Data structures')
+@docs_group('Charging')
 @dataclass
 class ActorPricingInfo:
     """Result of the `ChargingManager.get_pricing_info` method."""
@@ -90,7 +98,6 @@ class ActorPricingInfo:
     """Price of every known event type."""
 
 
-@ignore_docs
 class ChargingManagerImplementation(ChargingManager):
     """Implementation of the `ChargingManager` Protocol - this is only meant to be instantiated internally."""
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_configuration.py

@@ -25,7 +25,7 @@ def _transform_to_list(value: Any) -> list[str] | None:
     return value if isinstance(value, list) else str(value).split(',')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_crypto.py

@@ -12,7 +12,6 @@ from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding, rsa
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
-from apify_shared.utils import ignore_docs
 from crawlee._utils.crypto import crypto_random_object_id
 
 from apify._consts import ENCRYPTED_INPUT_VALUE_REGEXP, ENCRYPTED_JSON_VALUE_PREFIX, ENCRYPTED_STRING_VALUE_PREFIX
@@ -22,7 +21,6 @@ ENCRYPTION_IV_LENGTH = 16
 ENCRYPTION_AUTH_TAG_LENGTH = 16
 
 
-@ignore_docs
 def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     """Encrypts the given value using AES cipher and the password for encryption using the public key.
 
@@ -66,7 +64,6 @@ def public_encrypt(value: str, *, public_key: rsa.RSAPublicKey) -> dict:
     }
 
 
-@ignore_docs
 def private_decrypt(
     encrypted_password: str,
     encrypted_value: str,
@@ -118,7 +115,6 @@ def private_decrypt(
     return decipher_bytes.decode('utf-8')
 
 
-@ignore_docs
 def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey:
     private_key = serialization.load_pem_private_key(
         base64.b64decode(private_key_file_base64.encode('utf-8')),
@@ -138,7 +134,6 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey:
     return public_key
 
 
-@ignore_docs
 def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any:
     """Decrypt input secrets."""
     if not isinstance(input_data, dict):
@@ -180,7 +175,6 @@ def encode_base62(num: int) -> str:
     return res
 
 
-@ignore_docs
 def create_hmac_signature(secret_key: str, message: str) -> str:
     """Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_models.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
     from typing import TypeAlias
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -35,14 +35,14 @@ class Webhook(BaseModel):
     ] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -63,7 +63,7 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -74,7 +74,7 @@ class ActorRunOptions(BaseModel):
     max_total_charge_usd: Annotated[Decimal | None, Field(alias='maxTotalChargeUsd')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -92,7 +92,7 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
-@docs_group('Data structures')
+@docs_group('Actor')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_platform_event_manager.py

@@ -27,17 +27,10 @@ if TYPE_CHECKING:
 
     from apify._configuration import Configuration
 
-
 __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
 
 
-@docs_group('Data structures')
-class PersistStateEvent(BaseModel):
-    name: Literal[Event.PERSIST_STATE]
-    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
-
-
-@docs_group('Data structures')
+@docs_group('Event data')
 class SystemInfoEventData(BaseModel):
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
@@ -64,31 +57,37 @@ class SystemInfoEventData(BaseModel):
         )
 
 
-@docs_group('Data structures')
+@docs_group('Events')
+class PersistStateEvent(BaseModel):
+    name: Literal[Event.PERSIST_STATE]
+    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+
+
+@docs_group('Events')
 class SystemInfoEvent(BaseModel):
     name: Literal[Event.SYSTEM_INFO]
     data: SystemInfoEventData
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class MigratingEvent(BaseModel):
     name: Literal[Event.MIGRATING]
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class AbortingEvent(BaseModel):
     name: Literal[Event.ABORTING]
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class ExitEvent(BaseModel):
     name: Literal[Event.EXIT]
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class EventWithoutData(BaseModel):
     name: Literal[
         Event.SESSION_RETIRED,
@@ -101,13 +100,13 @@ class EventWithoutData(BaseModel):
     data: Any = None
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class DeprecatedEvent(BaseModel):
     name: Literal['cpuInfo']
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
 
 
-@docs_group('Data structures')
+@docs_group('Events')
 class UnknownEvent(BaseModel):
     name: str
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
@@ -120,7 +119,7 @@ event_data_adapter = TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent](
 )
 
 
-@docs_group('Classes')
+@docs_group('Event managers')
 class PlatformEventManager(EventManager):
     """A class for managing Actor events.
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_proxy_configuration.py

@@ -10,7 +10,6 @@ from urllib.parse import urljoin, urlparse
 import httpx
 
 from apify_shared.consts import ApifyEnvVars
-from apify_shared.utils import ignore_docs
 from crawlee.proxy_configuration import ProxyConfiguration as CrawleeProxyConfiguration
 from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
 from crawlee.proxy_configuration import _NewUrlFunction
@@ -28,7 +27,6 @@ COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')
 SESSION_ID_MAX_LENGTH = 50
 
 
-@ignore_docs
 def is_url(url: str) -> bool:
     """Check if the given string is a valid URL."""
     try:
@@ -69,7 +67,7 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -89,7 +87,7 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
-@docs_group('Classes')
+@docs_group('Configuration')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 
@@ -104,7 +102,6 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
 
     _configuration: Configuration
 
-    @ignore_docs
     def __init__(
         self,
         *,

{apify-2.7.0 → apify-2.7.0b2}/src/apify/_utils.py

@@ -30,7 +30,19 @@ def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
 
 
-GroupName = Literal['Classes', 'Abstract classes', 'Interfaces', 'Data structures', 'Errors', 'Functions']
+# The order of the rendered API groups is defined in the website/docusaurus.config.js file.
+GroupName = Literal[
+    'Actor',
+    'Charging',
+    'Configuration',
+    'Event data',
+    'Event managers',
+    'Events',
+    'Request loaders',
+    'Storage clients',
+    'Storage data',
+    'Storages',
+]
 
 
 def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001

{apify-2.7.0 → apify-2.7.0b2}/src/apify/apify_storage_client/_apify_storage_client.py

@@ -20,7 +20,7 @@ if TYPE_CHECKING:
     from apify._configuration import Configuration
 
 
-@docs_group('Classes')
+@docs_group('Storage clients')
 class ApifyStorageClient(StorageClient):
     """A storage client implementation based on the Apify platform storage."""
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/log.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 
-from apify_shared.utils import ignore_docs
 from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
 
 # Name of the logger used throughout the library (resolves to 'apify')
@@ -12,7 +11,6 @@ logger_name = __name__.split('.')[0]
 logger = logging.getLogger(logger_name)
 
 
-@ignore_docs
 class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 (Inherited from parent class)
     pass
 

{apify-2.7.0 → apify-2.7.0b2}/src/apify/storages/_request_list.py

@@ -38,7 +38,7 @@ class _SimpleUrlInput(_RequestDetails):
 url_input_adapter = TypeAdapter(list[_RequestsFromUrlInput | _SimpleUrlInput])
 
 
-@docs_group('Classes')
+@docs_group('Request loaders')
 class RequestList(CrawleeRequestList):
     """Extends crawlee RequestList.
 

{apify-2.7.0 → apify-2.7.0b2}/tests/integration/README.md

@@ -126,3 +126,26 @@ async def test_something(
 
     assert actor_run.status == 'SUCCEEDED'
 ```
+
+### Asserts
+
+Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown, without the left and right values. This means, we must include explicit assertion messages to aid potential debugging.
+
+```python
+async def test_add_and_fetch_requests(
+    make_actor: MakeActorFunction,
+    run_actor: RunActorFunction,
+) -> None:
+    """Test basic functionality of adding and fetching requests."""
+
+    async def main() -> None:
+        async with Actor:
+            rq = await Actor.open_request_queue()
+            await rq.add_request(f'https://apify.com/')
+            assert is_finished is False, f'is_finished={is_finished}'
+
+    actor = await make_actor(label='rq-test', main_func=main)
+    run_result = await run_actor(actor)
+
+    assert run_result.status == 'SUCCEEDED'
+```