apify 2.7.3__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

apify/events/_types.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field
+
+from crawlee.events._types import (
+    Event,
+    EventAbortingData,
+    EventExitData,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+)
+
+from apify._utils import docs_group
+
+
+@docs_group('Event data')
+class SystemInfoEventData(BaseModel):
+    mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    created_at: Annotated[datetime, Field(alias='createdAt')]
+
+    def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
+        return EventSystemInfoData.model_validate(
+            {
+                'cpu_info': {
+                    'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
+                    'created_at': self.created_at,
+                },
+                'memory_info': {
+                    'total_size': self.mem_max_bytes,
+                    'current_size': self.mem_current_bytes,
+                    'created_at': self.created_at,
+                },
+            }
+        )
+
+
+@docs_group('Events')
+class PersistStateEvent(BaseModel):
+    name: Literal[Event.PERSIST_STATE]
+    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+
+
+@docs_group('Events')
+class SystemInfoEvent(BaseModel):
+    name: Literal[Event.SYSTEM_INFO]
+    data: SystemInfoEventData
+
+
+@docs_group('Events')
+class MigratingEvent(BaseModel):
+    name: Literal[Event.MIGRATING]
+    data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+
+
+@docs_group('Events')
+class AbortingEvent(BaseModel):
+    name: Literal[Event.ABORTING]
+    data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+
+
+@docs_group('Events')
+class ExitEvent(BaseModel):
+    name: Literal[Event.EXIT]
+    data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+
+
+@docs_group('Events')
+class EventWithoutData(BaseModel):
+    name: Literal[
+        Event.SESSION_RETIRED,
+        Event.BROWSER_LAUNCHED,
+        Event.BROWSER_RETIRED,
+        Event.BROWSER_CLOSED,
+        Event.PAGE_CREATED,
+        Event.PAGE_CLOSED,
+    ]
+    data: Any = None
+
+
+@docs_group('Events')
+class DeprecatedEvent(BaseModel):
+    name: Literal['cpuInfo']
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+@docs_group('Events')
+class UnknownEvent(BaseModel):
+    name: str
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData

apify/log.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 
-from apify_shared.utils import ignore_docs
 from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
 
 # Name of the logger used throughout the library (resolves to 'apify')
@@ -12,7 +11,6 @@ logger_name = __name__.split('.')[0]
 logger = logging.getLogger(logger_name)
 
 
-@ignore_docs
 class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 (Inherited from parent class)
     pass
 
@@ -29,13 +27,6 @@ def _configure_logging() -> None:
     else:
         apify_client_logger.setLevel(level)
 
-    # Silence HTTPX logger unless debug logging is requested
-    httpx_logger = logging.getLogger('httpx')
-    if level > logging.DEBUG:
-        httpx_logger.setLevel(logging.WARNING)
-    else:
-        httpx_logger.setLevel(level)
-
     # Use configured log level for apify logger
     apify_logger = logging.getLogger('apify')
     configure_logger(apify_logger, remove_old_handlers=True)

apify/request_loaders/__init__.py

@@ -0,0 +1,18 @@
+from crawlee.request_loaders import (
+    RequestList,
+    RequestLoader,
+    RequestManager,
+    RequestManagerTandem,
+    SitemapRequestLoader,
+)
+
+from ._apify_request_list import ApifyRequestList
+
+__all__ = [
+    'ApifyRequestList',
+    'RequestList',
+    'RequestLoader',
+    'RequestManager',
+    'RequestManagerTandem',
+    'SitemapRequestLoader',
+]

apify/{storages/_request_list.py → request_loaders/_apify_request_list.py}

@@ -3,16 +3,15 @@ from __future__ import annotations
 import asyncio
 import re
 from asyncio import Task
-from functools import partial
-from typing import Annotated, Any, Union
+from typing import Annotated, Any
 
 from pydantic import BaseModel, Field, TypeAdapter
 
-from crawlee import Request
 from crawlee._types import HttpMethod
-from crawlee.http_clients import HttpClient, HttpxHttpClient
-from crawlee.request_loaders import RequestList as CrawleeRequestList
+from crawlee.http_clients import HttpClient, ImpitHttpClient
+from crawlee.request_loaders import RequestList
 
+from apify import Request
 from apify._utils import docs_group
 
 URL_NO_COMMAS_REGEX = re.compile(
@@ -35,11 +34,11 @@ class _SimpleUrlInput(_RequestDetails):
     url: str
 
 
-url_input_adapter = TypeAdapter(list[Union[_RequestsFromUrlInput, _SimpleUrlInput]])
+url_input_adapter = TypeAdapter(list[_RequestsFromUrlInput | _SimpleUrlInput])
 
 
-@docs_group('Classes')
-class RequestList(CrawleeRequestList):
+@docs_group('Request loaders')
+class ApifyRequestList(RequestList):
     """Extends crawlee RequestList.
 
     Method open is used to create RequestList from actor's requestListSources input.
@@ -50,7 +49,7 @@ class RequestList(CrawleeRequestList):
         name: str | None = None,
         request_list_sources_input: list[dict[str, Any]] | None = None,
         http_client: HttpClient | None = None,
-    ) -> RequestList:
+    ) -> ApifyRequestList:
         """Initialize a new instance from request list source input.
 
         Args:
@@ -74,24 +73,26 @@ class RequestList(CrawleeRequestList):
         ```
         """
         request_list_sources_input = request_list_sources_input or []
-        return await RequestList._create_request_list(name, request_list_sources_input, http_client)
+        return await ApifyRequestList._create_request_list(name, request_list_sources_input, http_client)
 
     @staticmethod
     async def _create_request_list(
         name: str | None, request_list_sources_input: list[dict[str, Any]], http_client: HttpClient | None
-    ) -> RequestList:
+    ) -> ApifyRequestList:
         if not http_client:
-            http_client = HttpxHttpClient()
+            http_client = ImpitHttpClient()
 
         url_inputs = url_input_adapter.validate_python(request_list_sources_input)
 
         simple_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _SimpleUrlInput)]
         remote_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _RequestsFromUrlInput)]
 
-        simple_url_requests = RequestList._create_requests_from_input(simple_url_inputs)
-        remote_url_requests = await RequestList._fetch_requests_from_url(remote_url_inputs, http_client=http_client)
+        simple_url_requests = ApifyRequestList._create_requests_from_input(simple_url_inputs)
+        remote_url_requests = await ApifyRequestList._fetch_requests_from_url(
+            remote_url_inputs, http_client=http_client
+        )
 
-        return RequestList(name=name, requests=simple_url_requests + remote_url_requests)
+        return ApifyRequestList(name=name, requests=simple_url_requests + remote_url_requests)
 
     @staticmethod
     def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:
@@ -119,13 +120,15 @@ class RequestList(CrawleeRequestList):
         """
         created_requests: list[Request] = []
 
-        def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
+        async def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
             """Extract links from response body and use them to create `Request` objects.
 
             Use the regular expression to find all matching links in the response body, then create `Request`
             objects from these links and the provided input attributes.
             """
-            matches = re.finditer(URL_NO_COMMAS_REGEX, task.result().read().decode('utf-8'))
+            response = await (task.result()).read()
+            matches = re.finditer(URL_NO_COMMAS_REGEX, response.decode('utf-8'))
+
             created_requests.extend(
                 [
                     Request.from_url(
@@ -148,7 +151,11 @@ class RequestList(CrawleeRequestList):
                 )
             )
 
-            get_response_task.add_done_callback(partial(create_requests_from_response, remote_url_requests_input))
+            get_response_task.add_done_callback(
+                lambda task, inp=remote_url_requests_input: asyncio.create_task(  # type: ignore[misc]
+                    create_requests_from_response(inp, task)
+                )
+            )
             remote_url_requests.append(get_response_task)
 
         await asyncio.gather(*remote_url_requests)

apify/scrapy/_logging_config.py

@@ -10,7 +10,7 @@ from apify.log import ActorLogFormatter
 
 # Define logger names.
 _PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
-_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'protego', 'twisted']
 _ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
 
 
@@ -37,9 +37,6 @@ def initialize_logging() -> None:
     for logger_name in [None, *_ALL_LOGGERS]:
         _configure_logger(logger_name, logging_level, handler)
 
-    # Set the 'httpx' logger to a less verbose level.
-    logging.getLogger('httpx').setLevel('WARNING')
-
     # Monkey-patch Scrapy's logging configuration to re-apply our settings.
     original_configure_logging = scrapy_logging.configure_logging
 

apify/scrapy/extensions/_httpcache.py

@@ -13,8 +13,8 @@ from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
 from apify.scrapy._async_thread import AsyncThread
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import KeyValueStore
 
 if TYPE_CHECKING:
@@ -51,10 +51,14 @@ class ApifyCacheStorage:
         kvs_name = get_kvs_name(spider.name)
 
         async def open_kvs() -> KeyValueStore:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await KeyValueStore.open(name=kvs_name, storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await KeyValueStore.open(
+                    name=kvs_name,
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await KeyValueStore.open(name=kvs_name)
 
         logger.debug("Starting background thread for cache storage's event loop")

apify/scrapy/requests.py

@@ -10,9 +10,10 @@ from scrapy import Spider
 from scrapy.http.headers import Headers
 from scrapy.utils.request import request_from_dict
 
-from crawlee import Request as ApifyRequest
 from crawlee._types import HttpHeaders
 
+from apify import Request as ApifyRequest
+
 logger = getLogger(__name__)
 
 
@@ -121,7 +122,7 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
 
         # Update the meta field with the meta field from the apify_request
         meta = scrapy_request.meta or {}
-        meta.update({'apify_request_id': apify_request.id, 'apify_request_unique_key': apify_request.unique_key})
+        meta.update({'apify_request_unique_key': apify_request.unique_key})
         # scrapy_request.meta is a property, so we have to set it like this
         scrapy_request._meta = meta  # noqa: SLF001
 
@@ -133,7 +134,6 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
             url=apify_request.url,
             method=apify_request.method,
             meta={
-                'apify_request_id': apify_request.id,
                 'apify_request_unique_key': apify_request.unique_key,
             },
         )

apify/scrapy/scheduler.py

@@ -11,7 +11,7 @@ from scrapy.utils.reactor import is_asyncio_reactor_installed
 from ._async_thread import AsyncThread
 from .requests import to_apify_request, to_scrapy_request
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import RequestQueue
 
 if TYPE_CHECKING:
@@ -49,10 +49,13 @@ class ApifyScheduler(BaseScheduler):
         self.spider = spider
 
         async def open_rq() -> RequestQueue:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await RequestQueue.open(storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await RequestQueue.open(
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await RequestQueue.open()
 
         try:

apify/storage_clients/__init__.py

@@ -0,0 +1,10 @@
+from crawlee.storage_clients import MemoryStorageClient
+
+from ._apify import ApifyStorageClient
+from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
+
+__all__ = [
+    'ApifyStorageClient',
+    'FileSystemStorageClient',
+    'MemoryStorageClient',
+]

apify/storage_clients/_apify/__init__.py

@@ -0,0 +1,11 @@
+from ._dataset_client import ApifyDatasetClient
+from ._key_value_store_client import ApifyKeyValueStoreClient
+from ._request_queue_client import ApifyRequestQueueClient
+from ._storage_client import ApifyStorageClient
+
+__all__ = [
+    'ApifyDatasetClient',
+    'ApifyKeyValueStoreClient',
+    'ApifyRequestQueueClient',
+    'ApifyStorageClient',
+]