apify 1.7.3b4__tar.gz → 2.0.0__tar.gz

apify-2.0.0/PKG-INFO

@@ -0,0 +1,209 @@
+Metadata-Version: 2.1
+Name: apify
+Version: 2.0.0
+Summary: Apify SDK for Python
+License: Apache-2.0
+Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
+Author: Apify Technologies s.r.o.
+Author-email: support@apify.com
+Requires-Python: >=3.9,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Provides-Extra: scrapy
+Requires-Dist: apify-client (>=1.7.1)
+Requires-Dist: apify-shared (>=1.1.2)
+Requires-Dist: crawlee (>=0.3.5)
+Requires-Dist: cryptography (>=42.0.0)
+Requires-Dist: httpx (>=0.27.0)
+Requires-Dist: lazy-object-proxy (>=1.10.0)
+Requires-Dist: scrapy (>=2.11.0) ; extra == "scrapy"
+Requires-Dist: typing-extensions (>=4.1.0)
+Requires-Dist: websockets (>=10.0)
+Project-URL: Apify Homepage, https://apify.com
+Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
+Project-URL: Documentation, https://docs.apify.com/sdk/python/
+Project-URL: Homepage, https://docs.apify.com/sdk/python/
+Project-URL: Issue Tracker, https://github.com/apify/apify-sdk-python/issues
+Project-URL: Repository, https://github.com/apify/apify-sdk-python
+Description-Content-Type: text/markdown
+
+# Apify SDK for Python
+
+The Apify SDK for Python is the official library to create [Apify Actors](https://docs.apify.com/platform/actors)
+in Python. It provides useful features like Actor lifecycle management, local storage emulation, and Actor
+event handling.
+
+If you just need to access the [Apify API](https://docs.apify.com/api/v2) from your Python applications,
+check out the [Apify Client for Python](https://docs.apify.com/api/client/python) instead.
+
+## Installation
+
+The Apify SDK for Python is available on PyPI as the `apify` package.
+For default installation, using Pip, run the following:
+
+```bash
+pip install apify
+```
+
+For users interested in integrating Apify with Scrapy, we provide a package extra called `scrapy`.
+To install Apify with the `scrapy` extra, use the following command:
+
+```bash
+pip install apify[scrapy]
+```
+
+## Documentation
+
+For usage instructions, check the documentation on [Apify Docs](https://docs.apify.com/sdk/python/).
+
+## Examples
+
+Below are few examples demonstrating how to use the Apify SDK with some web scraping-related libraries.
+
+### Apify SDK with HTTPX and BeautifulSoup
+
+This example illustrates how to integrate the Apify SDK with [HTTPX](https://www.python-httpx.org/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) to scrape data from web pages.
+
+```python
+from apify import Actor
+from bs4 import BeautifulSoup
+from httpx import AsyncClient
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}])
+
+        # Open the default request queue for handling URLs to be processed.
+        request_queue = await Actor.open_request_queue()
+
+        # Enqueue the start URLs.
+        for start_url in start_urls:
+            url = start_url.get('url')
+            await request_queue.add_request(url)
+
+        # Process the URLs from the request queue.
+        while request := await request_queue.fetch_next_request():
+            Actor.log.info(f'Scraping {request.url} ...')
+
+            # Fetch the HTTP response from the specified URL using HTTPX.
+            async with AsyncClient() as client:
+                response = await client.get(request.url)
+
+            # Parse the HTML content using Beautiful Soup.
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # Extract the desired data.
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string,
+                'h1s': [h1.text for h1 in soup.find_all('h1')],
+                'h2s': [h2.text for h2 in soup.find_all('h2')],
+                'h3s': [h3.text for h3 in soup.find_all('h3')],
+            }
+
+            # Store the extracted data to the default dataset.
+            await Actor.push_data(data)
+```
+
+### Apify SDK with PlaywrightCrawler from Crawlee
+
+This example demonstrates how to use the Apify SDK alongside `PlaywrightCrawler` from [Crawlee](https://crawlee.dev/python) to perform web scraping.
+
+```python
+from apify import Actor, Request
+from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = [url.get('url') for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}])]
+
+        # Exit if no start URLs are provided.
+        if not start_urls:
+            Actor.log.info('No start URLs specified in Actor input, exiting...')
+            await Actor.exit()
+
+        # Create a crawler.
+        crawler = PlaywrightCrawler(
+            # Limit the crawl to max requests. Remove or increase it for crawling all links.
+            max_requests_per_crawl=50,
+            headless=True,
+        )
+
+        # Define a request handler, which will be called for every request.
+        @crawler.router.default_handler
+        async def request_handler(context: PlaywrightCrawlingContext) -> None:
+            url = context.request.url
+            Actor.log.info(f'Scraping {url}...')
+
+            # Extract the desired data.
+            data = {
+                'url': context.request.url,
+                'title': await context.page.title(),
+                'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()],
+                'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()],
+                'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()],
+            }
+
+            # Store the extracted data to the default dataset.
+            await context.push_data(data)
+
+            # Enqueue additional links found on the current page.
+            await context.enqueue_links()
+
+        # Run the crawler with the starting URLs.
+        await crawler.run(start_urls)
+```
+
+## What are Actors?
+
+Actors are serverless cloud programs that can do almost anything a human can do in a web browser.
+They can do anything from small tasks such as filling in forms or unsubscribing from online services,
+all the way up to scraping and processing vast numbers of web pages.
+
+They can be run either locally, or on the [Apify platform](https://docs.apify.com/platform/),
+where you can run them at scale, monitor them, schedule them, or publish and monetize them.
+
+If you're new to Apify, learn [what is Apify](https://docs.apify.com/platform/about)
+in the Apify platform documentation.
+
+## Creating Actors
+
+To create and run Actors through Apify Console,
+see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template).
+
+To create and run Python Actors locally, check the documentation for
+[how to create and run Python Actors locally](https://docs.apify.com/sdk/python/docs/overview/running-locally).
+
+## Guides
+
+To see how you can use the Apify SDK with other popular libraries used for web scraping,
+check out our guides for using
+[Requests and HTTPX](https://docs.apify.com/sdk/python/docs/guides/requests-and-httpx),
+[Beautiful Soup](https://docs.apify.com/sdk/python/docs/guides/beautiful-soup),
+[Playwright](https://docs.apify.com/sdk/python/docs/guides/playwright),
+[Selenium](https://docs.apify.com/sdk/python/docs/guides/selenium),
+or [Scrapy](https://docs.apify.com/sdk/python/docs/guides/scrapy).
+
+## Usage concepts
+
+To learn more about the features of the Apify SDK and how to use them,
+check out the Usage Concepts section in the sidebar,
+particularly the guides for the [Actor lifecycle](https://docs.apify.com/sdk/python/docs/concepts/actor-lifecycle),
+[working with storages](https://docs.apify.com/sdk/python/docs/concepts/storages),
+[handling Actor events](https://docs.apify.com/sdk/python/docs/concepts/actor-events)
+or [how to use proxies](https://docs.apify.com/sdk/python/docs/concepts/proxy-management).
+

apify-2.0.0/README.md

@@ -0,0 +1,171 @@
+# Apify SDK for Python
+
+The Apify SDK for Python is the official library to create [Apify Actors](https://docs.apify.com/platform/actors)
+in Python. It provides useful features like Actor lifecycle management, local storage emulation, and Actor
+event handling.
+
+If you just need to access the [Apify API](https://docs.apify.com/api/v2) from your Python applications,
+check out the [Apify Client for Python](https://docs.apify.com/api/client/python) instead.
+
+## Installation
+
+The Apify SDK for Python is available on PyPI as the `apify` package.
+For default installation, using Pip, run the following:
+
+```bash
+pip install apify
+```
+
+For users interested in integrating Apify with Scrapy, we provide a package extra called `scrapy`.
+To install Apify with the `scrapy` extra, use the following command:
+
+```bash
+pip install apify[scrapy]
+```
+
+## Documentation
+
+For usage instructions, check the documentation on [Apify Docs](https://docs.apify.com/sdk/python/).
+
+## Examples
+
+Below are few examples demonstrating how to use the Apify SDK with some web scraping-related libraries.
+
+### Apify SDK with HTTPX and BeautifulSoup
+
+This example illustrates how to integrate the Apify SDK with [HTTPX](https://www.python-httpx.org/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) to scrape data from web pages.
+
+```python
+from apify import Actor
+from bs4 import BeautifulSoup
+from httpx import AsyncClient
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}])
+
+        # Open the default request queue for handling URLs to be processed.
+        request_queue = await Actor.open_request_queue()
+
+        # Enqueue the start URLs.
+        for start_url in start_urls:
+            url = start_url.get('url')
+            await request_queue.add_request(url)
+
+        # Process the URLs from the request queue.
+        while request := await request_queue.fetch_next_request():
+            Actor.log.info(f'Scraping {request.url} ...')
+
+            # Fetch the HTTP response from the specified URL using HTTPX.
+            async with AsyncClient() as client:
+                response = await client.get(request.url)
+
+            # Parse the HTML content using Beautiful Soup.
+            soup = BeautifulSoup(response.content, 'html.parser')
+
+            # Extract the desired data.
+            data = {
+                'url': actor_input['url'],
+                'title': soup.title.string,
+                'h1s': [h1.text for h1 in soup.find_all('h1')],
+                'h2s': [h2.text for h2 in soup.find_all('h2')],
+                'h3s': [h3.text for h3 in soup.find_all('h3')],
+            }
+
+            # Store the extracted data to the default dataset.
+            await Actor.push_data(data)
+```
+
+### Apify SDK with PlaywrightCrawler from Crawlee
+
+This example demonstrates how to use the Apify SDK alongside `PlaywrightCrawler` from [Crawlee](https://crawlee.dev/python) to perform web scraping.
+
+```python
+from apify import Actor, Request
+from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+
+
+async def main() -> None:
+    async with Actor:
+        # Retrieve the Actor input, and use default values if not provided.
+        actor_input = await Actor.get_input() or {}
+        start_urls = [url.get('url') for url in actor_input.get('start_urls', [{'url': 'https://apify.com'}])]
+
+        # Exit if no start URLs are provided.
+        if not start_urls:
+            Actor.log.info('No start URLs specified in Actor input, exiting...')
+            await Actor.exit()
+
+        # Create a crawler.
+        crawler = PlaywrightCrawler(
+            # Limit the crawl to max requests. Remove or increase it for crawling all links.
+            max_requests_per_crawl=50,
+            headless=True,
+        )
+
+        # Define a request handler, which will be called for every request.
+        @crawler.router.default_handler
+        async def request_handler(context: PlaywrightCrawlingContext) -> None:
+            url = context.request.url
+            Actor.log.info(f'Scraping {url}...')
+
+            # Extract the desired data.
+            data = {
+                'url': context.request.url,
+                'title': await context.page.title(),
+                'h1s': [await h1.text_content() for h1 in await context.page.locator('h1').all()],
+                'h2s': [await h2.text_content() for h2 in await context.page.locator('h2').all()],
+                'h3s': [await h3.text_content() for h3 in await context.page.locator('h3').all()],
+            }
+
+            # Store the extracted data to the default dataset.
+            await context.push_data(data)
+
+            # Enqueue additional links found on the current page.
+            await context.enqueue_links()
+
+        # Run the crawler with the starting URLs.
+        await crawler.run(start_urls)
+```
+
+## What are Actors?
+
+Actors are serverless cloud programs that can do almost anything a human can do in a web browser.
+They can do anything from small tasks such as filling in forms or unsubscribing from online services,
+all the way up to scraping and processing vast numbers of web pages.
+
+They can be run either locally, or on the [Apify platform](https://docs.apify.com/platform/),
+where you can run them at scale, monitor them, schedule them, or publish and monetize them.
+
+If you're new to Apify, learn [what is Apify](https://docs.apify.com/platform/about)
+in the Apify platform documentation.
+
+## Creating Actors
+
+To create and run Actors through Apify Console,
+see the [Console documentation](https://docs.apify.com/academy/getting-started/creating-actors#choose-your-template).
+
+To create and run Python Actors locally, check the documentation for
+[how to create and run Python Actors locally](https://docs.apify.com/sdk/python/docs/overview/running-locally).
+
+## Guides
+
+To see how you can use the Apify SDK with other popular libraries used for web scraping,
+check out our guides for using
+[Requests and HTTPX](https://docs.apify.com/sdk/python/docs/guides/requests-and-httpx),
+[Beautiful Soup](https://docs.apify.com/sdk/python/docs/guides/beautiful-soup),
+[Playwright](https://docs.apify.com/sdk/python/docs/guides/playwright),
+[Selenium](https://docs.apify.com/sdk/python/docs/guides/selenium),
+or [Scrapy](https://docs.apify.com/sdk/python/docs/guides/scrapy).
+
+## Usage concepts
+
+To learn more about the features of the Apify SDK and how to use them,
+check out the Usage Concepts section in the sidebar,
+particularly the guides for the [Actor lifecycle](https://docs.apify.com/sdk/python/docs/concepts/actor-lifecycle),
+[working with storages](https://docs.apify.com/sdk/python/docs/concepts/storages),
+[handling Actor events](https://docs.apify.com/sdk/python/docs/concepts/actor-events)
+or [how to use proxies](https://docs.apify.com/sdk/python/docs/concepts/proxy-management).

apify-2.0.0/pyproject.toml

@@ -0,0 +1,199 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "apify"
+version = "2.0.0"
+description = "Apify SDK for Python"
+authors = ["Apify Technologies s.r.o. <support@apify.com>"]
+license = "Apache-2.0"
+readme = "README.md"
+packages = [{ include = "apify", from = "src" }]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries",
+]
+keywords = [
+    "apify",
+    "sdk",
+    "automation",
+    "chrome",
+    "crawlee",
+    "crawler",
+    "headless",
+    "scraper",
+    "scraping",
+]
+
+[tool.poetry.urls]
+"Homepage" = "https://docs.apify.com/sdk/python/"
+"Apify Homepage" = "https://apify.com"
+"Changelog" = "https://docs.apify.com/sdk/python/docs/changelog"
+"Documentation" = "https://docs.apify.com/sdk/python/"
+"Issue Tracker" = "https://github.com/apify/apify-sdk-python/issues"
+"Repository" = "https://github.com/apify/apify-sdk-python"
+
+# We use inclusive ordered comparison clauses for external packages intentionally in order to enhance SDK's
+# compatibility with external packages. This decision was discussed in detail in the following PR:
+# https://github.com/apify/apify-sdk-python/pull/154.
+[tool.poetry.dependencies]
+python = "^3.9"
+apify-client = ">=1.7.1"
+apify-shared = ">=1.1.2"
+crawlee = ">=0.3.5"
+cryptography = ">=42.0.0"
+httpx = ">=0.27.0"
+lazy-object-proxy = ">=1.10.0"
+scrapy = { version = ">=2.11.0", optional = true }
+typing-extensions = ">=4.1.0"
+websockets = ">=10.0"
+
+[tool.poetry.group.dev.dependencies]
+build = "~1.2.0"
+filelock = "~3.16.0"
+griffe = "~1.2.0"
+mypy = "~1.11.0"
+pre-commit = "~3.8.0"
+pydoc-markdown = "~4.8.0"
+pytest = "~8.3.0"
+pytest-asyncio = "~0.24.0"
+pytest-cov = "~5.0.0"
+pytest-only = "~2.1.0"
+pytest-timeout = "~2.3.0"
+pytest-xdist = "~3.6.0"
+respx = "~0.21.0"
+ruff = "~0.6.0"
+setuptools = "~74.0.0"     # setuptools are used by pytest but not explicitly required
+
+[tool.poetry.extras]
+scrapy = ["scrapy"]
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "ANN101",   # Missing type annotation for `self` in method
+    "ANN102",   # Missing type annotation for `{name}` in classmethod
+    "ANN401",   # Dynamically typed expressions (typing.Any) are disallowed in {filename}
+    "ASYNC109", # Async function definition with a `timeout` parameter
+    "BLE001",   # Do not catch blind exception
+    "C901",     # `{name}` is too complex
+    "COM812",   # This rule may cause conflicts when used with the formatter
+    "D100",     # Missing docstring in public module
+    "D104",     # Missing docstring in public package
+    "D107",     # Missing docstring in `__init__`
+    "EM",       # flake8-errmsg
+    "G004",     # Logging statement uses f-string
+    "ISC001",   # This rule may cause conflicts when used with the formatter
+    "FIX",      # flake8-fixme
+    "PGH003",   # Use specific rule codes when ignoring type issues
+    "PLR0911",  # Too many return statements
+    "PLR0913",  # Too many arguments in function definition
+    "PLR0915",  # Too many statements
+    "PTH",      # flake8-use-pathlib
+    "PYI034",   # `__aenter__` methods in classes like `{name}` usually return `self` at runtime
+    "PYI036",   # The second argument in `__aexit__` should be annotated with `object` or `BaseException | None`
+    "S102",     # Use of `exec` detected
+    "S105",     # Possible hardcoded password assigned to
+    "S106",     # Possible hardcoded password assigned to argument: "{name}"
+    "S301",     # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
+    "S303",     # Use of insecure MD2, MD4, MD5, or SHA1 hash function
+    "S311",     # Standard pseudo-random generators are not suitable for cryptographic purposes
+    "TD002",    # Missing author in TODO; try: `# TODO(<author_name>): ...` or `# TODO @<author_name>: ...
+    "TRY003",   # Avoid specifying long messages outside the exception class
+]
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+
+[tool.ruff.lint.per-file-ignores]
+"**/__init__.py" = [
+    "F401", # Unused imports
+]
+"**/{scripts}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "T20",     # flake8-print
+]
+"**/{tests}/*" = [
+    "D",       # Everything from the pydocstyle
+    "INP001",  # File {filename} is part of an implicit namespace package, add an __init__.py
+    "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable
+    "S101",    # Use of assert detected
+    "SLF001",  # Private member accessed: `{name}`
+    "T20",     # flake8-print
+    "TRY301",  # Abstract `raise` to an inner function
+    "TID252",  # Prefer absolute imports over relative imports from parent modules
+]
+"**/{docs}/**" = [
+    "D",      # Everything from the pydocstyle
+    "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py
+    "F841",   # Local variable {variable} is assigned to but never used
+]
+
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "single"
+
+[tool.ruff.lint.flake8-builtins]
+builtins-ignorelist = ["id"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.isort]
+known-local-folder = ["apify"]
+known-first-party = ["apify_client", "apify_shared", "crawlee"]
+
+[tool.ruff.lint.pylint]
+max-branches = 18
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+asyncio_mode = "auto"
+timeout = 1200
+
+[tool.mypy]
+python_version = "3.9"
+plugins = ["pydantic.mypy"]
+files = ["scripts", "src", "tests"]
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_calls = true
+disallow_untyped_decorators = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_return_any = true
+warn_unreachable = true
+warn_unused_ignores = true
+exclude = []
+
+[[tool.mypy.overrides]]
+module = ['scrapy', 'scrapy.*', 'lazy_object_proxy']
+ignore_missing_imports = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "assert_never()"
+]
+
+[tool.basedpyright]
+typeCheckingMode = "standard"
+
+[tool.ipdb]
+context = 7

apify-2.0.0/src/apify/__init__.py

@@ -0,0 +1,24 @@
+from importlib import metadata
+
+from apify_shared.consts import WebhookEventType
+from crawlee import Request
+from crawlee.events._types import Event
+
+from apify._actor import Actor
+from apify._configuration import Configuration
+from apify._models import Webhook
+from apify._proxy_configuration import ProxyConfiguration, ProxyInfo
+
+__version__ = metadata.version('apify')
+
+__all__ = [
+    'Actor',
+    'Event',
+    'Configuration',
+    'ProxyConfiguration',
+    'ProxyInfo',
+    'Request',
+    'Webhook',
+    'WebhookEventType',
+    '__version__',
+]