apify 2.1.0__py3-none-any.whl → 2.1.0b1__py3-none-any.whl

apify/__init__.py

@@ -13,8 +13,8 @@ __version__ = metadata.version('apify')
 
 __all__ = [
     'Actor',
-    'Configuration',
     'Event',
+    'Configuration',
     'ProxyConfiguration',
     'ProxyInfo',
     'Request',

apify/_actor.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
 
 from lazy_object_proxy import Proxy
 from pydantic import AliasChoices
+from typing_extensions import Self
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
@@ -21,7 +22,7 @@ from apify._crypto import decrypt_input_secrets, load_private_key
 from apify._models import ActorRun
 from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
-from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
+from apify._utils import get_system_info, is_running_in_ipython
 from apify.apify_storage_client import ApifyStorageClient
 from apify.log import _configure_logging, logger
 from apify.storages import Dataset, KeyValueStore, RequestQueue
@@ -30,8 +31,6 @@ if TYPE_CHECKING:
     import logging
     from types import TracebackType
 
-    from typing_extensions import Self
-
     from crawlee.proxy_configuration import _NewUrlFunction
 
     from apify._models import Webhook
@@ -40,8 +39,6 @@ if TYPE_CHECKING:
 MainReturnType = TypeVar('MainReturnType')
 
 
-@docs_name('Actor')
-@docs_group('Classes')
 class _ActorType:
     """The class of `Actor`. Only make a new instance if you're absolutely sure you need to."""
 

apify/_configuration.py

@@ -1,3 +1,4 @@
+# ruff: noqa: TCH001 TCH002 TCH003 (so that pydantic annotations work)
 from __future__ import annotations
 
 from datetime import datetime, timedelta
@@ -10,10 +11,7 @@ from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
 
-from apify._utils import docs_group
 
-
-@docs_group('Classes')
 class Configuration(CrawleeConfiguration):
     """A class for specifying the configuration of an Actor.
 
@@ -253,7 +251,6 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Date when the Actor will time out',
         ),
-        BeforeValidator(lambda val: val if val != '' else None),  # We should accept empty environment variables as well
     ] = None
 
     standby_port: Annotated[
@@ -323,4 +320,4 @@ class Configuration(CrawleeConfiguration):
 
 
 # Monkey-patch the base class so that it works with the extended configuration
-CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]
+CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore

apify/_models.py

@@ -1,3 +1,4 @@
+# ruff: noqa: TCH001 TCH002 TCH003 (Pydantic)
 from __future__ import annotations
 
 from datetime import datetime, timedelta
@@ -9,10 +10,7 @@ from apify_shared.consts import ActorJobStatus, MetaOrigin, WebhookEventType
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 
-from apify._utils import docs_group
 
-
-@docs_group('Data structures')
 class Webhook(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -31,14 +29,12 @@ class Webhook(BaseModel):
     ] = None
 
 
-@docs_group('Data structures')
 class ActorRunMeta(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
     origin: Annotated[MetaOrigin, Field()]
 
 
-@docs_group('Data structures')
 class ActorRunStats(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -59,7 +55,6 @@ class ActorRunStats(BaseModel):
     compute_units: Annotated[float, Field(alias='computeUnits')]
 
 
-@docs_group('Data structures')
 class ActorRunOptions(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -69,7 +64,6 @@ class ActorRunOptions(BaseModel):
     disk_mbytes: Annotated[int, Field(alias='diskMbytes')]
 
 
-@docs_group('Data structures')
 class ActorRunUsage(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 
@@ -87,7 +81,6 @@ class ActorRunUsage(BaseModel):
     proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None
 
 
-@docs_group('Data structures')
 class ActorRun(BaseModel):
     __model_config__ = ConfigDict(populate_by_name=True)
 

apify/_platform_event_manager.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-from datetime import datetime
+from datetime import datetime  # noqa: TCH003
 from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
 
 import websockets.client
@@ -19,7 +19,6 @@ from crawlee.events._types import (
     EventSystemInfoData,
 )
 
-from apify._utils import docs_group
 from apify.log import logger
 
 if TYPE_CHECKING:
@@ -31,13 +30,11 @@ if TYPE_CHECKING:
 __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
 
 
-@docs_group('Data structures')
 class PersistStateEvent(BaseModel):
     name: Literal[Event.PERSIST_STATE]
     data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
 
 
-@docs_group('Data structures')
 class SystemInfoEventData(BaseModel):
     mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
     mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
@@ -52,7 +49,7 @@ class SystemInfoEventData(BaseModel):
         return EventSystemInfoData.model_validate(
             {
                 'cpu_info': {
-                    'used_ratio': self.cpu_current_usage / 100,
+                    'used_ratio': self.cpu_current_usage,
                     'created_at': self.created_at,
                 },
                 'memory_info': {
@@ -64,31 +61,26 @@ class SystemInfoEventData(BaseModel):
         )
 
 
-@docs_group('Data structures')
 class SystemInfoEvent(BaseModel):
     name: Literal[Event.SYSTEM_INFO]
     data: SystemInfoEventData
 
 
-@docs_group('Data structures')
 class MigratingEvent(BaseModel):
     name: Literal[Event.MIGRATING]
     data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
 
 
-@docs_group('Data structures')
 class AbortingEvent(BaseModel):
     name: Literal[Event.ABORTING]
     data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
 
 
-@docs_group('Data structures')
 class ExitEvent(BaseModel):
     name: Literal[Event.EXIT]
     data: Annotated[EventExitData, Field(default_factory=EventExitData)]
 
 
-@docs_group('Data structures')
 class EventWithoutData(BaseModel):
     name: Literal[
         Event.SESSION_RETIRED,
@@ -101,13 +93,11 @@ class EventWithoutData(BaseModel):
     data: Any = None
 
 
-@docs_group('Data structures')
 class DeprecatedEvent(BaseModel):
     name: Literal['cpuInfo']
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
 
 
-@docs_group('Data structures')
 class UnknownEvent(BaseModel):
     name: str
     data: Annotated[dict[str, Any], Field(default_factory=dict)]
@@ -135,7 +125,6 @@ event_data_adapter: TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent] =
 )
 
 
-@docs_group('Classes')
 class PlatformEventManager(EventManager):
     """A class for managing Actor events.
 

apify/_proxy_configuration.py

@@ -16,7 +16,6 @@ from crawlee.proxy_configuration import ProxyInfo as CrawleeProxyInfo
 from crawlee.proxy_configuration import _NewUrlFunction
 
 from apify._configuration import Configuration
-from apify._utils import docs_group
 from apify.log import logger
 
 if TYPE_CHECKING:
@@ -69,7 +68,6 @@ def _check(
         raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}')
 
 
-@docs_group('Classes')
 @dataclass
 class ProxyInfo(CrawleeProxyInfo):
     """Provides information about a proxy connection that is used for requests."""
@@ -89,7 +87,6 @@ class ProxyInfo(CrawleeProxyInfo):
     """
 
 
-@docs_group('Classes')
 class ProxyConfiguration(CrawleeProxyConfiguration):
     """Configures a connection to a proxy server with the provided options.
 
@@ -280,7 +277,7 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return
 
         status = None
-        async with httpx.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
+        async with httpx.AsyncClient(proxies=proxy_info.url, timeout=10) as client:
             for _ in range(2):
                 try:
                     response = await client.get(proxy_status_url)

apify/_utils.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import builtins
 import sys
 from importlib import metadata
-from typing import Callable, Literal
 
 
 def get_system_info() -> dict:
@@ -12,7 +11,6 @@ def get_system_info() -> dict:
     system_info: dict[str, str | bool] = {
         'apify_sdk_version': metadata.version('apify'),
         'apify_client_version': metadata.version('apify-client'),
-        'crawlee_version': metadata.version('crawlee'),
         'python_version': python_version,
         'os': sys.platform,
     }
@@ -25,34 +23,3 @@ def get_system_info() -> dict:
 
 def is_running_in_ipython() -> bool:
     return getattr(builtins, '__IPYTHON__', False)
-
-
-GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions']
-
-
-def docs_group(group_name: GroupName) -> Callable:  # noqa: ARG001
-    """Decorator to mark symbols for rendering and grouping in documentation.
-
-    This decorator is used purely for documentation purposes and does not alter the behavior
-    of the decorated callable.
-    """
-
-    def wrapper(func: Callable) -> Callable:
-        return func
-
-    return wrapper
-
-
-def docs_name(symbol_name: str) -> Callable:  # noqa: ARG001
-    """Decorator for renaming symbols in documentation.
-
-    This changes the rendered name of the symbol only in the rendered web documentation.
-
-    This decorator is used purely for documentation purposes and does not alter the behavior
-    of the decorated callable.
-    """
-
-    def wrapper(func: Callable) -> Callable:
-        return func
-
-    return wrapper

apify/apify_storage_client/_apify_storage_client.py

@@ -5,7 +5,6 @@ from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.base_storage_client import BaseStorageClient
 
 from apify._configuration import Configuration
-from apify._utils import docs_group
 from apify.apify_storage_client._dataset_client import DatasetClient
 from apify.apify_storage_client._dataset_collection_client import DatasetCollectionClient
 from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient
@@ -14,7 +13,6 @@ from apify.apify_storage_client._request_queue_client import RequestQueueClient
 from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient
 
 
-@docs_group('Classes')
 class ApifyStorageClient(BaseStorageClient):
     """A storage client implementation based on the Apify platform storage."""
 

apify/apify_storage_client/_dataset_client.py

@@ -91,7 +91,7 @@ class DatasetClient(BaseDatasetClient):
         skip_empty: bool = False,
         skip_hidden: bool = False,
     ) -> AsyncIterator[dict]:
-        async for item in self._client.iterate_items(
+        return self._client.iterate_items(
             offset=offset,
             limit=limit,
             clean=clean,
@@ -101,8 +101,7 @@ class DatasetClient(BaseDatasetClient):
             unwind=unwind,
             skip_empty=skip_empty,
             skip_hidden=skip_hidden,
-        ):
-            yield item
+        )
 
     @override
     async def get_items_as_bytes(

apify/log.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(logger_name)
 
 
 @ignore_docs
-class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 (Inherited from parent class)
+class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
     pass
 
 

apify/scrapy/__init__.py

@@ -3,9 +3,9 @@ from apify.scrapy.scheduler import ApifyScheduler
 from apify.scrapy.utils import get_basic_auth_header, get_running_event_loop_id
 
 __all__ = [
+    'to_apify_request',
+    'to_scrapy_request',
     'ApifyScheduler',
     'get_basic_auth_header',
     'get_running_event_loop_id',
-    'to_apify_request',
-    'to_scrapy_request',
 ]

apify/scrapy/middlewares/apify_proxy.py

@@ -1,13 +1,11 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
 try:
-    if TYPE_CHECKING:
-        from scrapy import Request, Spider
-        from scrapy.crawler import Crawler
+    from scrapy import Request, Spider  # noqa: TCH002
     from scrapy.core.downloader.handlers.http11 import TunnelError
+    from scrapy.crawler import Crawler  # noqa: TCH002
     from scrapy.exceptions import NotConfigured
 except ImportError as exc:
     raise ImportError(
@@ -29,7 +27,7 @@ class ApifyHttpProxyMiddleware:
     proxy_settings = {'useApifyProxy': true, 'apifyProxyGroups': []}
     """
 
-    def __init__(self, proxy_settings: dict) -> None:
+    def __init__(self: ApifyHttpProxyMiddleware, proxy_settings: dict) -> None:
         """Create a new instance.
 
         Args:
@@ -68,7 +66,7 @@ class ApifyHttpProxyMiddleware:
 
         return cls(proxy_settings)
 
-    async def process_request(self, request: Request, spider: Spider) -> None:
+    async def process_request(self: ApifyHttpProxyMiddleware, request: Request, spider: Spider) -> None:
         """Process a Scrapy request by assigning a new proxy.
 
         Args:
@@ -91,7 +89,7 @@ class ApifyHttpProxyMiddleware:
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: updated request.meta={request.meta}')
 
     def process_exception(
-        self,
+        self: ApifyHttpProxyMiddleware,
         request: Request,
         exception: Exception,
         spider: Spider,
@@ -118,7 +116,7 @@ class ApifyHttpProxyMiddleware:
                 'reason="{exception}", skipping...'
             )
 
-    async def _get_new_proxy_url(self) -> ParseResult:
+    async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
         """Get a new proxy URL.
 
         Raises:

apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,12 +1,9 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 from itemadapter.adapter import ItemAdapter
 
 try:
-    if TYPE_CHECKING:
-        from scrapy import Item, Spider
+    from scrapy import Item, Spider  # noqa: TCH002
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
@@ -22,7 +19,7 @@ class ActorDatasetPushPipeline:
     """
 
     async def process_item(
-        self,
+        self: ActorDatasetPushPipeline,
         item: Item,
         spider: Spider,
     ) -> Item:

apify/scrapy/requests.py

@@ -42,10 +42,8 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
     Returns:
         The converted Apify request if the conversion was successful, otherwise None.
     """
-    if not isinstance(scrapy_request, Request):
-        Actor.log.warning(  # type: ignore[unreachable]
-            'Failed to convert to Apify request: Scrapy request must be a Request instance.'
-        )
+    if not isinstance(cast(Any, scrapy_request), Request):
+        Actor.log.warning('Failed to convert to Apify request: Scrapy request must be a Request instance.')
         return None
 
     call_id = crypto_random_object_id(8)
@@ -55,7 +53,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
         if _is_request_produced_by_middleware(scrapy_request):
             unique_key = compute_unique_key(
                 url=scrapy_request.url,
-                method=scrapy_request.method,  # type: ignore[arg-type]  # str vs literal
+                method=scrapy_request.method,
                 payload=scrapy_request.body,
                 use_extended_unique_key=True,
             )
@@ -82,9 +80,9 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
 
         # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
         if isinstance(scrapy_request.headers, Headers):
-            apify_request.headers = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
+            apify_request.headers = HttpHeaders(scrapy_request.headers.to_unicode_dict())
         else:
-            Actor.log.warning(  # type: ignore[unreachable]
+            Actor.log.warning(
                 f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
             )
 

apify/scrapy/scheduler.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import traceback
-from typing import TYPE_CHECKING
 
 from apify._configuration import Configuration
 from apify.apify_storage_client import ApifyStorageClient
@@ -9,10 +8,8 @@ from apify.apify_storage_client import ApifyStorageClient
 try:
     from scrapy import Spider
     from scrapy.core.scheduler import BaseScheduler
+    from scrapy.http.request import Request  # noqa: TCH002
     from scrapy.utils.reactor import is_asyncio_reactor_installed
-
-    if TYPE_CHECKING:
-        from scrapy.http.request import Request
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
@@ -32,7 +29,7 @@ class ApifyScheduler(BaseScheduler):
     This scheduler requires the asyncio Twisted reactor to be installed.
     """
 
-    def __init__(self) -> None:
+    def __init__(self: ApifyScheduler) -> None:
         """Create a new instance."""
         if not is_asyncio_reactor_installed():
             raise ValueError(
@@ -43,7 +40,7 @@ class ApifyScheduler(BaseScheduler):
         self._rq: RequestQueue | None = None
         self.spider: Spider | None = None
 
-    def open(self, spider: Spider) -> None:  # this has to be named "open"
+    def open(self: ApifyScheduler, spider: Spider) -> None:  # this has to be named "open"
         """Open the scheduler.
 
         Args:
@@ -61,7 +58,7 @@ class ApifyScheduler(BaseScheduler):
             traceback.print_exc()
             raise
 
-    def has_pending_requests(self) -> bool:
+    def has_pending_requests(self: ApifyScheduler) -> bool:
         """Check if the scheduler has any pending requests.
 
         Returns:
@@ -78,7 +75,7 @@ class ApifyScheduler(BaseScheduler):
 
         return not is_finished
 
-    def enqueue_request(self, request: Request) -> bool:
+    def enqueue_request(self: ApifyScheduler, request: Request) -> bool:
         """Add a request to the scheduler.
 
         This could be called from either from a spider or a downloader middleware (e.g. redirect, retry, ...).
@@ -114,7 +111,7 @@ class ApifyScheduler(BaseScheduler):
         Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...')
         return bool(result.was_already_present)
 
-    def next_request(self) -> Request | None:
+    def next_request(self: ApifyScheduler) -> Request | None:
         """Fetch the next request from the scheduler.
 
         Returns:

apify/scrapy/utils.py

@@ -2,17 +2,14 @@ from __future__ import annotations
 
 import asyncio
 from base64 import b64encode
-from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
 from apify_shared.utils import ignore_docs
 
 try:
+    from scrapy.settings import Settings  # noqa: TCH002
     from scrapy.utils.project import get_project_settings
     from scrapy.utils.python import to_bytes
-
-    if TYPE_CHECKING:
-        from scrapy.settings import Settings
 except ImportError as exc:
     raise ImportError(
         'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '

apify/storages/__init__.py

@@ -1,5 +1,3 @@
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-from ._request_list import RequestList
-
-__all__ = ['Dataset', 'KeyValueStore', 'RequestList', 'RequestQueue']
+__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue']

{apify-2.1.0.dist-info → apify-2.1.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 2.1.0
+Version: 2.1.0b1
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -21,13 +21,13 @@ Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
 Requires-Dist: apify-client (>=1.8.1)
 Requires-Dist: apify-shared (>=1.1.2)
-Requires-Dist: crawlee (>=0.4.0,<0.5.0)
+Requires-Dist: crawlee (>=0.3.8)
 Requires-Dist: cryptography (>=42.0.0)
-Requires-Dist: httpx (>=0.27.0,<0.28.0)
+Requires-Dist: httpx (>=0.27.0)
 Requires-Dist: lazy-object-proxy (>=1.10.0)
 Requires-Dist: scrapy (>=2.11.0) ; extra == "scrapy"
 Requires-Dist: typing-extensions (>=4.1.0)
-Requires-Dist: websockets (>=10.0,<14.0.0)
+Requires-Dist: websockets (>=10.0)
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
 Project-URL: Documentation, https://docs.apify.com/sdk/python/

apify-2.1.0b1.dist-info/RECORD

@@ -0,0 +1,37 @@
+apify/__init__.py,sha256=ikoi2EpDYl6y-XSVtlU8UsdQdMEyOiIJCRRAaZFDOP8,550
+apify/_actor.py,sha256=oPgQ3rxxIEzVcZ9XtI3lf1a_6gwIMgxihNuYGjJpGww,41816
+apify/_configuration.py,sha256=fK-BmsdctMJJT93SsupG98_DmknPnfifmIp5pwehYlA,9508
+apify/_consts.py,sha256=_Xq4hOfOA1iZ3n1P967YWdyncKivpbX6RTlp_qanUoE,330
+apify/_crypto.py,sha256=e0_aM3l9_5Osk-jszYOOjrAKK60OggSHbiw5c30QnsU,5638
+apify/_models.py,sha256=oYlTEr-DyQAE-V2rrYD5PhUxTXVPdAig7QV-u6CJw3E,5571
+apify/_platform_event_manager.py,sha256=K4cHabbQ7_ex7vkX-c-VhAOp8Efw3HDn5Wp4lfA-qAU,7571
+apify/_proxy_configuration.py,sha256=2z4VV_NrnIp6pDpgQKlKpcHM2pPyXiOpFedpPWje48A,13087
+apify/_utils.py,sha256=x4lnR9RNulySiEQTft-GeQqUcJsRr0k8p0Sv9NTeWFg,638
+apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
+apify/apify_storage_client/_apify_storage_client.py,sha256=xi4OFchxhe-1-sykanH6Zcya4OcBhn2uf7OQ1pV4Ins,2338
+apify/apify_storage_client/_dataset_client.py,sha256=j9seF2OKvbSMD9R9XF9fpa1vtr_1w4JcRV--WCmvU4E,5501
+apify/apify_storage_client/_dataset_collection_client.py,sha256=fkYvYGQCigHD2CDzpWk0swNAkfvAinAhMGpYqllle3E,1445
+apify/apify_storage_client/_key_value_store_client.py,sha256=uyeQgb75sGFsqIS4sq4hEZ3QP81COLfS3tmTqHc0tso,3340
+apify/apify_storage_client/_key_value_store_collection_client.py,sha256=vCtMTI-jx89Qp5WHILDNkCthwLuv0MAwm1J_5E4aypU,1519
+apify/apify_storage_client/_request_queue_client.py,sha256=jAiFkaJ38_myHFGTw-Rk21wmpbN0UCR2w2SFoimFGFc,5826
+apify/apify_storage_client/_request_queue_collection_client.py,sha256=NnO73UJ9ZrjV8xoudo30wfaM-SojRkG0guhxDyB-K1g,1527
+apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify/log.py,sha256=zIVjrqQ1DNWNQQOAmdmR9oAbf4nJH7CSMB6u4OOUf6I,1448
+apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify/scrapy/__init__.py,sha256=qDPV_zTRFaUqoFOyS5g4uBfz-UCkmWYJ82VXQ_3Cw6k,348
+apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
+apify/scrapy/middlewares/apify_proxy.py,sha256=_1WO7NKHxIcPf8mSNjsqANTEsx7ygMTuRQW9fbwKMO8,5837
+apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
+apify/scrapy/pipelines/actor_dataset_push.py,sha256=QERmmExQOGIKQ70-p-lCj5qyE-c-fnYplEqd4mgaB1Q,953
+apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify/scrapy/requests.py,sha256=F4VNaX2fGqybJKbhcRcz0_m6dXse5LzKll4gtMuTRko,7480
+apify/scrapy/scheduler.py,sha256=AAIKY5i1QxkC1mtmix6n3M2eQaOw-d1T56Noue9xToc,6013
+apify/scrapy/utils.py,sha256=0XdFxi1qlUa6gHXG96e1FU9gW0N5Rsu0sVZklFYfC2U,2884
+apify/storages/__init__.py,sha256=-9tEYJVabVs_eRVhUehxN58GH0UG8OfuGjGwuDieP2M,122
+apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+apify-2.1.0b1.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-2.1.0b1.dist-info/METADATA,sha256=wP9WixgCbxLoefOJF57CcIMtBCtJb70JQJBAPNCMSNE,8657
+apify-2.1.0b1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+apify-2.1.0b1.dist-info/RECORD,,

apify/storages/_request_list.py

@@ -1,150 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import re
-from asyncio import Task
-from functools import partial
-from typing import Annotated, Any, Union
-
-from pydantic import BaseModel, Field, TypeAdapter
-
-from crawlee import Request
-from crawlee._types import HttpMethod
-from crawlee.http_clients import BaseHttpClient, HttpxHttpClient
-from crawlee.storages import RequestList as CrawleeRequestList
-
-from apify._utils import docs_group
-
-URL_NO_COMMAS_REGEX = re.compile(
-    r'https?:\/\/(www\.)?([^\W_]|[^\W_][-\w0-9@:%._+~#=]{0,254}[^\W_])\.[a-z]{2,63}(:\d{1,5})?(\/[-\w@:%+.~#?&/=()]*)?'
-)
-
-
-class _RequestDetails(BaseModel):
-    method: HttpMethod = 'GET'
-    payload: str = ''
-    headers: Annotated[dict[str, str], Field(default_factory=dict)] = {}
-    user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')] = {}
-
-
-class _RequestsFromUrlInput(_RequestDetails):
-    requests_from_url: str = Field(alias='requestsFromUrl')
-
-
-class _SimpleUrlInput(_RequestDetails):
-    url: str
-
-
-url_input_adapter = TypeAdapter(list[Union[_RequestsFromUrlInput, _SimpleUrlInput]])
-
-
-@docs_group('Classes')
-class RequestList(CrawleeRequestList):
-    """Extends crawlee RequestList.
-
-    Method open is used to create RequestList from actor's requestListSources input.
-    """
-
-    @staticmethod
-    async def open(
-        name: str | None = None,
-        request_list_sources_input: list[dict[str, Any]] | None = None,
-        http_client: BaseHttpClient | None = None,
-    ) -> RequestList:
-        """Creates RequestList from Actor input requestListSources.
-
-        Args:
-            name: Name of the returned RequestList.
-            request_list_sources_input: List of dicts with either url key or requestsFromUrl key.
-            http_client: Client that will be used to send get request to urls defined by value of requestsFromUrl keys.
-
-        Returns:
-            RequestList created from request_list_sources_input.
-
-        ### Usage
-
-        ```python
-        example_input = [
-            # Gather urls from response body.
-            {'requestsFromUrl': 'https://crawlee.dev/file.txt', 'method': 'GET'},
-            # Directly include this url.
-            {'url': 'https://crawlee.dev', 'method': 'GET'}
-        ]
-        request_list = await RequestList.open(request_list_sources_input=example_input)
-        ```
-        """
-        request_list_sources_input = request_list_sources_input or []
-        return await RequestList._create_request_list(name, request_list_sources_input, http_client)
-
-    @staticmethod
-    async def _create_request_list(
-        name: str | None, request_list_sources_input: list[dict[str, Any]], http_client: BaseHttpClient | None
-    ) -> RequestList:
-        if not http_client:
-            http_client = HttpxHttpClient()
-
-        url_inputs = url_input_adapter.validate_python(request_list_sources_input)
-
-        simple_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _SimpleUrlInput)]
-        remote_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _RequestsFromUrlInput)]
-
-        simple_url_requests = RequestList._create_requests_from_input(simple_url_inputs)
-        remote_url_requests = await RequestList._fetch_requests_from_url(remote_url_inputs, http_client=http_client)
-
-        return RequestList(name=name, requests=simple_url_requests + remote_url_requests)
-
-    @staticmethod
-    def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:
-        return [
-            Request.from_url(
-                method=request_input.method,
-                url=request_input.url,
-                payload=request_input.payload.encode('utf-8'),
-                headers=request_input.headers,
-                user_data=request_input.user_data,
-            )
-            for request_input in simple_url_inputs
-        ]
-
-    @staticmethod
-    async def _fetch_requests_from_url(
-        remote_url_requests_inputs: list[_RequestsFromUrlInput], http_client: BaseHttpClient
-    ) -> list[Request]:
-        """Crete list of requests from url.
-
-        Send GET requests to urls defined in each requests_from_url of remote_url_requests_inputs. Run extracting
-        callback on each response body and use URL_NO_COMMAS_REGEX regex to find all links. Create list of Requests from
-        collected links and additional inputs stored in other attributes of each remote_url_requests_inputs.
-        """
-        created_requests: list[Request] = []
-
-        def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
-            """Callback to scrape response body with regexp and create Requests from matches."""
-            matches = re.finditer(URL_NO_COMMAS_REGEX, task.result().read().decode('utf-8'))
-            created_requests.extend(
-                [
-                    Request.from_url(
-                        match.group(0),
-                        method=request_input.method,
-                        payload=request_input.payload.encode('utf-8'),
-                        headers=request_input.headers,
-                        user_data=request_input.user_data,
-                    )
-                    for match in matches
-                ]
-            )
-
-        remote_url_requests = []
-        for remote_url_requests_input in remote_url_requests_inputs:
-            get_response_task = asyncio.create_task(
-                http_client.send_request(
-                    method='GET',
-                    url=remote_url_requests_input.requests_from_url,
-                )
-            )
-
-            get_response_task.add_done_callback(partial(create_requests_from_response, remote_url_requests_input))
-            remote_url_requests.append(get_response_task)
-
-        await asyncio.gather(*remote_url_requests)
-        return created_requests

apify-2.1.0.dist-info/RECORD

@@ -1,38 +0,0 @@
-apify/__init__.py,sha256=99ynaDWBLEcCjdLq7R0Exy_iACsXiXoQ8VUZKmbzTeM,550
-apify/_actor.py,sha256=AUviY4qrX4UoN7fSZtXXSHqEk4rrQwBymMLjkgb4Mzg,41887
-apify/_configuration.py,sha256=WC9X5gQ-e5r4BGQw-Kaq77oxLlPx7vtq0M-L4VlKrIc,9633
-apify/_consts.py,sha256=_Xq4hOfOA1iZ3n1P967YWdyncKivpbX6RTlp_qanUoE,330
-apify/_crypto.py,sha256=e0_aM3l9_5Osk-jszYOOjrAKK60OggSHbiw5c30QnsU,5638
-apify/_models.py,sha256=Btlz-23obKY5tJ75JnUwkVNC2lmU1IEBbdU3HvWaVhg,5748
-apify/_platform_event_manager.py,sha256=44xyV0Lpzf4h4VZ0rkyYg_nhbQkEONNor8_Z9gIKO40,7899
-apify/_proxy_configuration.py,sha256=vdDiE5dfyNQYEXKXnj0jcgL3rG3-qanwNSybrVl1xT8,13167
-apify/_utils.py,sha256=CCLkpAsZKp00ykm88Z_Fbck5PNT0j6mJYOuD0RxzZUs,1620
-apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
-apify/apify_storage_client/_apify_storage_client.py,sha256=NsZBleJNHLBXVyG__bVjdCGEI30cnmVZngCbQaVekfk,2397
-apify/apify_storage_client/_dataset_client.py,sha256=FfXew6tBiZRkpovyPaQ__xhtZZ-rZvjijwBIIyRahH8,5536
-apify/apify_storage_client/_dataset_collection_client.py,sha256=fkYvYGQCigHD2CDzpWk0swNAkfvAinAhMGpYqllle3E,1445
-apify/apify_storage_client/_key_value_store_client.py,sha256=uyeQgb75sGFsqIS4sq4hEZ3QP81COLfS3tmTqHc0tso,3340
-apify/apify_storage_client/_key_value_store_collection_client.py,sha256=vCtMTI-jx89Qp5WHILDNkCthwLuv0MAwm1J_5E4aypU,1519
-apify/apify_storage_client/_request_queue_client.py,sha256=jAiFkaJ38_myHFGTw-Rk21wmpbN0UCR2w2SFoimFGFc,5826
-apify/apify_storage_client/_request_queue_collection_client.py,sha256=NnO73UJ9ZrjV8xoudo30wfaM-SojRkG0guhxDyB-K1g,1527
-apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/log.py,sha256=zElFyEp2RJN0kiHEwJhcjSCAuHrba5zYiq4pK2xsL_o,1450
-apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/__init__.py,sha256=HE5wCN7-DZKPydLCOvjNyLuL3CvN2fUFweXfrDfe1Ss,348
-apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
-apify/scrapy/middlewares/apify_proxy.py,sha256=9_-hJqTwQ4yVMjvN9zkJ_GXJADzrrYu8QoHZ6IX6fDs,5764
-apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
-apify/scrapy/pipelines/actor_dataset_push.py,sha256=otggoULfUdCqOPJLb9wMROZ9WylnlL-209930tMS2Rg,971
-apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/requests.py,sha256=yZ9hIsz2YyqOoOwzN9F1h76wG4qwawrI6h_6xq0I7Iw,7599
-apify/scrapy/scheduler.py,sha256=03kZxejWWb-TofJ-vpSZuQ28rT-qNjhhpC-QeO2OzoU,5977
-apify/scrapy/utils.py,sha256=758DcHCSAgCTProY0QX74uJ1XrzVsQwvCmFanj2f_3Q,2928
-apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
-apify/storages/_request_list.py,sha256=4nrvSdMUF-kiwGVIPEfIOygLKgjUpO37Jl8Om-jRbIU,5858
-apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-2.1.0.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-2.1.0.dist-info/METADATA,sha256=9K6Q_mrUCisKOlhMQuvt74xYMcaxu6UZPLMtec_8Pnc,8678
-apify-2.1.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-apify-2.1.0.dist-info/RECORD,,