apify 2.0.0b13__tar.gz → 2.0.0b15__tar.gz

{apify-2.0.0b13 → apify-2.0.0b15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 2.0.0b13
+Version: 2.0.0b15
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -20,7 +20,7 @@ Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
 Requires-Dist: apify-client (>=1.7.1)
 Requires-Dist: apify-shared (>=1.1.2)
-Requires-Dist: crawlee (>=0.3.0)
+Requires-Dist: crawlee (>=0.3.5)
 Requires-Dist: cryptography (>=42.0.0)
 Requires-Dist: httpx (>=0.27.0)
 Requires-Dist: lazy-object-proxy (>=1.10.0)
@@ -64,27 +64,108 @@ pip install apify[scrapy]
 
 For usage instructions, check the documentation on [Apify Docs](https://docs.apify.com/sdk/python/).
 
-## Example
+## Examples
+
+Below are few examples demonstrating how to use the Apify SDK with some web scraping-related libraries.
+
+### Apify SDK with HTTPX and BeautifulSoup
+
+This example illustrates how to integrate the Apify SDK with [HTTPX](https://www.python-httpx.org/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) to scrape data from web pages.
 
 ```python
 from apify import Actor
 from bs4 import BeautifulSoup
 from httpx import AsyncClient
 
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}])
+
+        # Open the default request queue for handling URLs to be processed.
+        request_queue = await Actor.open_request_queue()
+
+        # Enqueue the start URLs.
+        for start_url in start_urls:
+            url = start_url.get('url')
+            await request_queue.add_request(url)
+
+        # Process the URLs from the request queue.
+        while request := await request_queue.fetch_next_request():
+            Actor.log.info(f'Scraping {request.url} ...')
+
+            # Fetch the HTTP response from the specified URL using HTTPX.
+            async with AsyncClient() as client:
+                response = await client.get(request.url)
+
+            # Parse the HTML content using Beautiful Soup.
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # Extract the desired data.
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string,
+                'h1s': [h1.text for h1 in soup.find_all('h1')],
+                'h2s': [h2.text for h2 in soup.find_all('h2')],
+                'h3s': [h3.text for h3 in soup.find_all('h3')],
+            }
+
+            # Store the extracted data to the default dataset.
+            await Actor.push_data(data)
+```
+
+### Apify SDK with PlaywrightCrawler from Crawlee
+
+This example demonstrates how to use the Apify SDK alongside `PlaywrightCrawler` from [Crawlee](https://crawlee.dev/python) to perform web scraping.
+
+```python
+from apify import Actor, Request
+from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
 async def main() -> None:
     async with Actor:
-        # Read the input parameters from the Actor input
-        actor_input = await Actor.get_input()
-        # Fetch the HTTP response from the specified URL
-        async with AsyncClient() as client:
-            response = await client.get(actor_input['url'])
-        # Process the HTML content
-        soup = BeautifulSoup(response.content, 'html.parser')
-        # Push the extracted data
-        await Actor.push_data({
-            'url': actor_input['url'],
-            'title': soup.title.string,
-        })
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = [url.get('url') for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}])]
+
+        # Exit if no start URLs are provided.
+        if not start_urls:
+            Actor.log.info('No start URLs specified in Actor input, exiting...')
+            await Actor.exit()
+
+        # Create a crawler.
+        crawler = PlaywrightCrawler(
+            # Limit the crawl to max requests. Remove or increase it for crawling all links.
+            max_requests_per_crawl=50,
+            headless=True,
+        )
+
+        # Define a request handler, which will be called for every request.
+        @crawler.router.default_handler
+        async def request_handler(context: PlaywrightCrawlingContext) -> None:
+            url = context.request.url
+            Actor.log.info(f'Scraping {url}...')
+
+            # Extract the desired data.
+            data = {
+                'url': context.request.url,
+                'title': await context.page.title(),
+                'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()],
+                'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()],
+                'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()],
+            }
+
+            # Store the extracted data to the default dataset.
+            await context.push_data(data)
+
+            # Enqueue additional links found on the current page.
+            await context.enqueue_links()
+
+        # Run the crawler with the starting URLs.
+        await crawler.run(start_urls)
 ```
 
 ## What are Actors?

apify-2.0.0b15/README.md

@@ -0,0 +1,171 @@
+# Apify SDK for Python
+
+The Apify SDK for Python is the official library to create [Apify Actors](https://docs.apify.com/platform/actors)
+in Python. It provides useful features like Actor lifecycle management, local storage emulation, and Actor
+event handling.
+
+If you just need to access the [Apify API](https://docs.apify.com/api/v2) from your Python applications,
+check out the [Apify Client for Python](https://docs.apify.com/api/client/python) instead.
+
+## Installation
+
+The Apify SDK for Python is available on PyPI as the `apify` package.
+For default installation, using Pip, run the following:
+
+```bash
+pip install apify
+```
+
+For users interested in integrating Apify with Scrapy, we provide a package extra called `scrapy`.
+To install Apify with the `scrapy` extra, use the following command:
+
+```bash
+pip install apify[scrapy]
+```
+
+## Documentation
+
+For usage instructions, check the documentation on [Apify Docs](https://docs.apify.com/sdk/python/).
+
+## Examples
+
+Below are few examples demonstrating how to use the Apify SDK with some web scraping-related libraries.
+
+### Apify SDK with HTTPX and BeautifulSoup
+
+This example illustrates how to integrate the Apify SDK with [HTTPX](https://www.python-httpx.org/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) to scrape data from web pages.
+
+```python
+from apify import Actor
+from bs4 import BeautifulSoup
+from httpx import AsyncClient
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}])
+
+        # Open the default request queue for handling URLs to be processed.
+        request_queue = await Actor.open_request_queue()
+
+        # Enqueue the start URLs.
+        for start_url in start_urls:
+            url = start_url.get('url')
+            await request_queue.add_request(url)
+
+        # Process the URLs from the request queue.
+        while request := await request_queue.fetch_next_request():
+            Actor.log.info(f'Scraping {request.url} ...')
+
+            # Fetch the HTTP response from the specified URL using HTTPX.
+            async with AsyncClient() as client:
+                response = await client.get(request.url)
+
+            # Parse the HTML content using Beautiful Soup.
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # Extract the desired data.
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string,
+                'h1s': [h1.text for h1 in soup.find_all('h1')],
+                'h2s': [h2.text for h2 in soup.find_all('h2')],
+                'h3s': [h3.text for h3 in soup.find_all('h3')],
+            }
+
+            # Store the extracted data to the default dataset.
+            await Actor.push_data(data)
+```
+
+### Apify SDK with PlaywrightCrawler from Crawlee
+
+This example demonstrates how to use the Apify SDK alongside `PlaywrightCrawler` from [Crawlee](https://crawlee.dev/python) to perform web scraping.
+
+```python
+from apify import Actor, Request
+from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = [url.get('url') for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}])]
+
+        # Exit if no start URLs are provided.
+        if not start_urls:
+            Actor.log.info('No start URLs specified in Actor input, exiting...')
+            await Actor.exit()
+
+        # Create a crawler.
+        crawler = PlaywrightCrawler(
+            # Limit the crawl to max requests. Remove or increase it for crawling all links.
+            max_requests_per_crawl=50,
+            headless=True,
+        )
+
+        # Define a request handler, which will be called for every request.
+        @crawler.router.default_handler
+        async def request_handler(context: PlaywrightCrawlingContext) -> None:
+            url = context.request.url
+            Actor.log.info(f'Scraping {url}...')
+
+            # Extract the desired data.
+            data = {
+                'url': context.request.url,
+                'title': await context.page.title(),
+                'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()],
+                'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()],
+                'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()],
+            }
+
+            # Store the extracted data to the default dataset.
+            await context.push_data(data)
+
+            # Enqueue additional links found on the current page.
+            await context.enqueue_links()
+
+        # Run the crawler with the starting URLs.
+        await crawler.run(start_urls)
+```
+
+## What are Actors?
+
+Actors are serverless cloud programs that can do almost anything a human can do in a web browser.
+They can do anything from small tasks such as filling in forms or unsubscribing from online services,
+all the way up to scraping and processing vast numbers of web pages.
+
+They can be run either locally, or on the [Apify platform](https://docs.apify.com/platform/),
+where you can run them at scale, monitor them, schedule them, or publish and monetize them.
+
+If you're new to Apify, learn [what is Apify](https://docs.apify.com/platform/about)
+in the Apify platform documentation.
+
+## Creating Actors
+
+To create and run Actors through Apify Console,
+see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template).
+
+To create and run Python Actors locally, check the documentation for
+[how to create and run Python Actors locally](https://docs.apify.com/sdk/python/docs/overview/running-locally).
+
+## Guides
+
+To see how you can use the Apify SDK with other popular libraries used for web scraping,
+check out our guides for using
+[Requests and HTTPX](https://docs.apify.com/sdk/python/docs/guides/requests-and-httpx),
+[Beautiful Soup](https://docs.apify.com/sdk/python/docs/guides/beautiful-soup),
+[Playwright](https://docs.apify.com/sdk/python/docs/guides/playwright),
+[Selenium](https://docs.apify.com/sdk/python/docs/guides/selenium),
+or [Scrapy](https://docs.apify.com/sdk/python/docs/guides/scrapy).
+
+## Usage concepts
+
+To learn more about the features of the Apify SDK and how to use them,
+check out the Usage Concepts section in the sidebar,
+particularly the guides for the [Actor lifecycle](https://docs.apify.com/sdk/python/docs/concepts/actor-lifecycle),
+[working with storages](https://docs.apify.com/sdk/python/docs/concepts/storages),
+[handling Actor events](https://docs.apify.com/sdk/python/docs/concepts/actor-events)
+or [how to use proxies](https://docs.apify.com/sdk/python/docs/concepts/proxy-management).

{apify-2.0.0b13 → apify-2.0.0b15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify"
-version = "2.0.0b13"
+version = "2.0.0b15"
 description = "Apify SDK for Python"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 license = "Apache-2.0"
@@ -48,7 +48,7 @@ keywords = [
 python = "^3.9"
 apify-client = ">=1.7.1"
 apify-shared = ">=1.1.2"
-crawlee = ">=0.3.0"
+crawlee = ">=0.3.5"
 cryptography = ">=42.0.0"
 httpx = ">=0.27.0"
 lazy-object-proxy = ">=1.10.0"
@@ -58,7 +58,7 @@ websockets = ">=10.0"
 
 [tool.poetry.group.dev.dependencies]
 build = "~1.2.0"
-filelock = "~3.15.0"
+filelock = "~3.16.0"
 griffe = "~1.2.0"
 mypy = "~1.11.0"
 pre-commit = "~3.8.0"

{apify-2.0.0b13 → apify-2.0.0b15}/src/apify/_actor.py

@@ -24,7 +24,7 @@ from apify._platform_event_manager import EventManager, LocalEventManager, Platf
 from apify._proxy_configuration import ProxyConfiguration
 from apify._utils import get_system_info, is_running_in_ipython
 from apify.apify_storage_client import ApifyStorageClient
-from apify.log import logger
+from apify.log import _configure_logging, logger
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -46,16 +46,24 @@ class _ActorType:
     _configuration: Configuration
     _is_exiting = False
 
-    def __init__(self, config: Configuration | None = None) -> None:
+    def __init__(
+        self,
+        configuration: Configuration | None = None,
+        *,
+        configure_logging: bool = True,
+    ) -> None:
         """Create an Actor instance.
 
         Note that you don't have to do this, all the functionality is accessible using the default instance
         (e.g. `Actor.open_dataset()`).
 
         Args:
-            config: The Actor configuration to be used. If not passed, a new Configuration instance will be created.
+            configuration: The Actor configuration to be used. If not passed, a new Configuration instance will
+                be created.
+            configure_logging: Should the default logging configuration be configured?
         """
-        self._configuration = config or Configuration.get_global_configuration()
+        self._configuration = configuration or Configuration.get_global_configuration()
+        self._configure_logging = configure_logging
         self._apify_client = self.new_client()
 
         self._event_manager: EventManager
@@ -81,6 +89,9 @@ class _ActorType:
         When you exit the `async with` block, the `Actor.exit()` method is called, and if any exception happens while
         executing the block code, the `Actor.fail` method is called.
         """
+        if self._configure_logging:
+            _configure_logging(self._configuration)
+
         await self.init()
         return self
 
@@ -111,15 +122,20 @@ class _ActorType:
 
         return super().__repr__()
 
-    def __call__(self, config: Configuration) -> Self:
+    def __call__(self, configuration: Configuration | None = None, *, configure_logging: bool = True) -> Self:
         """Make a new Actor instance with a non-default configuration."""
-        return self.__class__(config=config)
+        return self.__class__(configuration=configuration, configure_logging=configure_logging)
 
     @property
     def apify_client(self) -> ApifyClientAsync:
         """The ApifyClientAsync instance the Actor instance uses."""
         return self._apify_client
 
+    @property
+    def configuration(self) -> Configuration:
+        """The Configuration instance the Actor instance uses."""
+        return self._configuration
+
     @property
     def config(self) -> Configuration:
         """The Configuration instance the Actor instance uses."""

apify-2.0.0b15/src/apify/log.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
+
+if TYPE_CHECKING:
+    from apify import Configuration
+
+# Name of the logger used throughout the library (resolves to 'apify')
+logger_name = __name__.split('.')[0]
+
+# Logger used throughout the library
+logger = logging.getLogger(logger_name)
+
+
+class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
+    pass
+
+
+def _configure_logging(configuration: Configuration) -> None:
+    apify_client_logger = logging.getLogger('apify_client')
+    configure_logger(apify_client_logger, configuration, remove_old_handlers=True)
+
+    level = get_configured_log_level(configuration)
+
+    # Keep apify_client logger quiet unless debug logging is requested
+    if level > logging.DEBUG:
+        apify_client_logger.setLevel(logging.INFO)
+    else:
+        apify_client_logger.setLevel(level)
+
+    # Silence HTTPX logger unless debug logging is requested
+    httpx_logger = logging.getLogger('httpx')
+    if level > logging.DEBUG:
+        httpx_logger.setLevel(logging.WARNING)
+    else:
+        httpx_logger.setLevel(level)
+
+    # Use configured log level for apify logger
+    apify_logger = logging.getLogger('apify')
+    configure_logger(apify_logger, configuration, remove_old_handlers=True)

apify-2.0.0b13/README.md

@@ -1,90 +0,0 @@
-# Apify SDK for Python
-
-The Apify SDK for Python is the official library to create [Apify Actors](https://docs.apify.com/platform/actors)
-in Python. It provides useful features like Actor lifecycle management, local storage emulation, and Actor
-event handling.
-
-If you just need to access the [Apify API](https://docs.apify.com/api/v2) from your Python applications,
-check out the [Apify Client for Python](https://docs.apify.com/api/client/python) instead.
-
-## Installation
-
-The Apify SDK for Python is available on PyPI as the `apify` package.
-For default installation, using Pip, run the following:
-
-```bash
-pip install apify
-```
-
-For users interested in integrating Apify with Scrapy, we provide a package extra called `scrapy`.
-To install Apify with the `scrapy` extra, use the following command:
-
-```bash
-pip install apify[scrapy]
-```
-
-## Documentation
-
-For usage instructions, check the documentation on [Apify Docs](https://docs.apify.com/sdk/python/).
-
-## Example
-
-```python
-from apify import Actor
-from bs4 import BeautifulSoup
-from httpx import AsyncClient
-
-async def main() -> None:
-    async with Actor:
-        # Read the input parameters from the Actor input
-        actor_input = await Actor.get_input()
-        # Fetch the HTTP response from the specified URL
-        async with AsyncClient() as client:
-            response = await client.get(actor_input['url'])
-        # Process the HTML content
-        soup = BeautifulSoup(response.content, 'html.parser')
-        # Push the extracted data
-        await Actor.push_data({
-            'url': actor_input['url'],
-            'title': soup.title.string,
-        })
-```
-
-## What are Actors?
-
-Actors are serverless cloud programs that can do almost anything a human can do in a web browser.
-They can do anything from small tasks such as filling in forms or unsubscribing from online services,
-all the way up to scraping and processing vast numbers of web pages.
-
-They can be run either locally, or on the [Apify platform](https://docs.apify.com/platform/),
-where you can run them at scale, monitor them, schedule them, or publish and monetize them.
-
-If you're new to Apify, learn [what is Apify](https://docs.apify.com/platform/about)
-in the Apify platform documentation.
-
-## Creating Actors
-
-To create and run Actors through Apify Console,
-see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template).
-
-To create and run Python Actors locally, check the documentation for
-[how to create and run Python Actors locally](https://docs.apify.com/sdk/python/docs/overview/running-locally).
-
-## Guides
-
-To see how you can use the Apify SDK with other popular libraries used for web scraping,
-check out our guides for using
-[Requests and HTTPX](https://docs.apify.com/sdk/python/docs/guides/requests-and-httpx),
-[Beautiful Soup](https://docs.apify.com/sdk/python/docs/guides/beautiful-soup),
-[Playwright](https://docs.apify.com/sdk/python/docs/guides/playwright),
-[Selenium](https://docs.apify.com/sdk/python/docs/guides/selenium),
-or [Scrapy](https://docs.apify.com/sdk/python/docs/guides/scrapy).
-
-## Usage concepts
-
-To learn more about the features of the Apify SDK and how to use them,
-check out the Usage Concepts section in the sidebar,
-particularly the guides for the [Actor lifecycle](https://docs.apify.com/sdk/python/docs/concepts/actor-lifecycle),
-[working with storages](https://docs.apify.com/sdk/python/docs/concepts/storages),
-[handling Actor events](https://docs.apify.com/sdk/python/docs/concepts/actor-events)
-or [how to use proxies](https://docs.apify.com/sdk/python/docs/concepts/proxy-management).

apify-2.0.0b13/src/apify/log.py

@@ -1,15 +0,0 @@
-from __future__ import annotations
-
-import logging
-
-from crawlee._log_config import CrawleeLogFormatter
-
-# Name of the logger used throughout the library (resolves to 'apify')
-logger_name = __name__.split('.')[0]
-
-# Logger used throughout the library
-logger = logging.getLogger(logger_name)
-
-
-class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
-    pass