apify 2.7.1b3__tar.gz → 2.7.1b5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/pre_release.yaml +5 -1
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/release.yaml +4 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/run_code_checks.yaml +5 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/CHANGELOG.md +214 -7
- {apify-2.7.1b3 → apify-2.7.1b5}/PKG-INFO +3 -3
- {apify-2.7.1b3 → apify-2.7.1b5}/pyproject.toml +7 -5
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/README.md +23 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/actor_source_base/Dockerfile +1 -1
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/actor_source_base/requirements.txt +2 -0
- apify-2.7.1b5/tests/integration/actor_source_base/server.py +101 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_api_helpers.py +2 -2
- apify-2.7.1b5/tests/integration/test_crawlers_with_storages.py +111 -0
- apify-2.7.1b5/tests/integration/test_request_queue.py +1192 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/uv.lock +330 -107
- {apify-2.7.1b3 → apify-2.7.1b5}/website/docusaurus.config.js +14 -1
- {apify-2.7.1b3 → apify-2.7.1b5}/website/package-lock.json +23 -9
- {apify-2.7.1b3 → apify-2.7.1b5}/website/package.json +6 -6
- apify-2.7.1b3/tests/integration/test_request_queue.py +0 -113
- {apify-2.7.1b3 → apify-2.7.1b5}/.editorconfig +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/CODEOWNERS +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/build_and_deploy_docs.yaml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/check_pr_title.yaml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.github/workflows/update_new_issue.yaml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.gitignore +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.markdownlint.yaml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/.pre-commit-config.yaml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/CONTRIBUTING.md +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/LICENSE +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/Makefile +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/README.md +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/01_introduction.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/02_running_actors_locally.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/03_actor_structure.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/code/01_introduction.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/code/actor_structure/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/code/actor_structure/__main__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/code/actor_structure/main.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/01_overview/code/actor_structure/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/01_beautifulsoup_httpx.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/02_crawlee.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/03_playwright.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/04_selenium.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/05_scrapy.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/01_beautifulsoup_httpx.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/02_crawlee_beautifulsoup.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/02_crawlee_playwright.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/03_playwright.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/04_selenium.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/__main__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/items.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/main.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/settings.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/spiders/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/spiders/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/02_guides/code/scrapy_project/src/spiders/title.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/01_actor_lifecycle.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/02_actor_input.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/03_storages.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/04_actor_events.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/05_proxy_management.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/06_interacting_with_other_actors.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/07_webhooks.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/08_access_apify_api.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/09_running_webserver.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/10_logging.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/11_configuration.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/12_pay_per_event.mdx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/01_context_manager.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/01_init_exit.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/01_reboot.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/01_status_message.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/02_input.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_dataset_exports.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_dataset_read_write.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_deleting_storages.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_kvs_iterating.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_kvs_public_url.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_kvs_read_write.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_opening_storages.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/03_rq.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/04_actor_events.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_apify_proxy.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_apify_proxy_config.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_custom_proxy.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_custom_proxy_function.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_proxy_actor_input.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_proxy_httpx.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/05_proxy_rotation.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/06_interacting_call.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/06_interacting_call_task.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/06_interacting_metamorph.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/06_interacting_start.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/07_webhook.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/07_webhook_preventing.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/08_actor_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/08_actor_new_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/09_webserver.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/10_log_config.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/10_logger_usage.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/10_redirect_log.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/10_redirect_log_existing_run.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/11_config.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/actor_charge.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/03_concepts/code/conditional_actor_charge.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/04_upgrading/upgrading_to_v2.md +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/docs/pyproject.toml +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/renovate.json +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_actor.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_charging.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_consts.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_crypto.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_models.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_platform_event_manager.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_proxy_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/_utils.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_apify_storage_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_dataset_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_dataset_collection_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_key_value_store_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_key_value_store_collection_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_request_queue_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/_request_queue_collection_client.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/apify_storage_client/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/log.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/_actor_runner.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/_async_thread.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/_logging_config.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/extensions/_httpcache.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/middlewares/apify_proxy.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/middlewares/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/pipelines/actor_dataset_push.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/pipelines/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/requests.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/scheduler.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/scrapy/utils.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/storages/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/storages/_request_list.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/src/apify/storages/py.typed +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/_utils.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/actor_source_base/src/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/actor_source_base/src/__main__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/actor_source_base/src/main.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/conftest.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_call_timeouts.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_charge.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_dataset.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_events.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_key_value_store.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_log.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_request_queue.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_actor_scrapy.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/integration/test_fixtures.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_dataset.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_env_helpers.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_helpers.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_key_value_store.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_log.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_non_default_instance.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_actor_request_queue.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/actor/test_request_list.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/conftest.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/extensions/test_httpcache.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/middlewares/test_apify_proxy.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/pipelines/test_actor_dataset_push.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/requests/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/requests/test_to_apify_request.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/requests/test_to_scrapy_request.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/utils/__init__.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/utils/test_apply_apify_settings.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/scrapy/utils/test_get_basic_auth_header.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/test_crypto.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/test_platform_event_manager.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/tests/unit/test_proxy_configuration.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/.eslintrc.json +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/babel.config.js +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/build_api_reference.sh +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/generate_module_shortcuts.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/sidebars.js +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/ApiLink.jsx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/Gradients.jsx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/Highlights.jsx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/Highlights.module.css +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/RunnableCodeBlock.jsx +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/components/RunnableCodeBlock.module.css +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/css/custom.css +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/pages/home_page_example.py +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/pages/index.js +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/src/pages/index.module.css +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/static/.nojekyll +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/static/img/docs-og.png +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/static/img/guides/redirected_logs_example.webp +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/tools/docs-prettier.config.js +0 -0
- {apify-2.7.1b3 → apify-2.7.1b5}/website/tools/utils/externalLink.js +0 -0
|
@@ -11,6 +11,10 @@ on:
|
|
|
11
11
|
# Or it can be triggered manually.
|
|
12
12
|
workflow_dispatch:
|
|
13
13
|
|
|
14
|
+
concurrency:
|
|
15
|
+
group: release
|
|
16
|
+
cancel-in-progress: false
|
|
17
|
+
|
|
14
18
|
jobs:
|
|
15
19
|
release_metadata:
|
|
16
20
|
if: "!startsWith(github.event.head_commit.message, 'docs') && !startsWith(github.event.head_commit.message, 'ci') && startsWith(github.repository, 'apify/')"
|
|
@@ -20,13 +24,13 @@ jobs:
|
|
|
20
24
|
version_number: ${{ steps.release_metadata.outputs.version_number }}
|
|
21
25
|
tag_name: ${{ steps.release_metadata.outputs.tag_name }}
|
|
22
26
|
changelog: ${{ steps.release_metadata.outputs.changelog }}
|
|
23
|
-
existing_changelog_path: CHANGELOG.md
|
|
24
27
|
steps:
|
|
25
28
|
- uses: apify/workflows/git-cliff-release@main
|
|
26
29
|
id: release_metadata
|
|
27
30
|
name: Prepare release metadata
|
|
28
31
|
with:
|
|
29
32
|
release_type: prerelease
|
|
33
|
+
existing_changelog_path: CHANGELOG.md
|
|
30
34
|
|
|
31
35
|
lint_check:
|
|
32
36
|
name: Lint check
|
|
@@ -6,6 +6,11 @@ on:
|
|
|
6
6
|
# step required for PRs from forks. This prevents their potential exposure.
|
|
7
7
|
pull_request:
|
|
8
8
|
|
|
9
|
+
# Trigger for pushing to the master branch is handled by the pre-release workflow.
|
|
10
|
+
|
|
11
|
+
# It should also be possible to trigger checks manually
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
|
|
9
14
|
jobs:
|
|
10
15
|
lint_check:
|
|
11
16
|
name: Lint check
|
|
@@ -2,11 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
<!-- git-cliff-unreleased-start -->
|
|
6
|
+
## 2.7.1 - **not yet released**
|
|
7
|
+
|
|
8
|
+
### 🐛 Bug Fixes
|
|
9
|
+
|
|
10
|
+
- Restrict apify-shared and apify-client versions ([#523](https://github.com/apify/apify-sdk-python/pull/523)) ([b3ae5a9](https://github.com/apify/apify-sdk-python/commit/b3ae5a972a65454a4998eda59c9fcc3f6b7e8579)) by [@vdusek](https://github.com/vdusek)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
<!-- git-cliff-unreleased-end -->
|
|
5
14
|
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
6
15
|
|
|
7
16
|
### 🚀 Features
|
|
8
17
|
|
|
9
|
-
- Expose `logger` argument on `Actor.call` to control log redirection from started Actor run ([#487](https://github.com/apify/apify-sdk-python/pull/487)) ([aa6fa47](https://github.com/apify/apify-sdk-python/commit/aa6fa4750ea1bc7909be1191c0d276a2046930c2)) by [@Pijukatel](https://github.com/Pijukatel)
|
|
10
18
|
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
11
19
|
|
|
12
20
|
### 🐛 Bug Fixes
|
|
@@ -15,6 +23,28 @@ All notable changes to this project will be documented in this file.
|
|
|
15
23
|
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
16
24
|
|
|
17
25
|
|
|
26
|
+
## [2.7.2](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.2) (2025-07-30)
|
|
27
|
+
|
|
28
|
+
### 🐛 Bug Fixes
|
|
29
|
+
|
|
30
|
+
- Restrict apify-shared and apify-client versions ([#523](https://github.com/apify/apify-sdk-python/pull/523)) ([581ebae](https://github.com/apify/apify-sdk-python/commit/581ebae5752a984a34cbabc02c49945ae392db00)) by [@vdusek](https://github.com/vdusek)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
## [2.7.1](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.1) (2025-07-24)
|
|
34
|
+
|
|
35
|
+
### 🐛 Bug Fixes
|
|
36
|
+
|
|
37
|
+
- Add back support for Python 3.9.
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
41
|
+
|
|
42
|
+
### 🚀 Features
|
|
43
|
+
|
|
44
|
+
- Expose `logger` argument on `Actor.call` to control log redirection from started Actor run ([#487](https://github.com/apify/apify-sdk-python/pull/487)) ([aa6fa47](https://github.com/apify/apify-sdk-python/commit/aa6fa4750ea1bc7909be1191c0d276a2046930c2)) by [@Pijukatel](https://github.com/Pijukatel)
|
|
45
|
+
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
46
|
+
|
|
47
|
+
|
|
18
48
|
## [2.6.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.6.0) (2025-06-09)
|
|
19
49
|
|
|
20
50
|
### 🚀 Features
|
|
@@ -143,33 +173,210 @@ All notable changes to this project will be documented in this file.
|
|
|
143
173
|
|
|
144
174
|
- [**breaking**] Preparation for v2 release ([#210](https://github.com/apify/apify-sdk-python/pull/210)) ([2f9dcc5](https://github.com/apify/apify-sdk-python/commit/2f9dcc559414f31e3f4fc87e72417a36494b9c84)) by [@janbuchar](https://github.com/janbuchar), closes [#135](https://github.com/apify/apify-sdk-python/issues/135), [#137](https://github.com/apify/apify-sdk-python/issues/137), [#138](https://github.com/apify/apify-sdk-python/issues/138), [#147](https://github.com/apify/apify-sdk-python/issues/147), [#149](https://github.com/apify/apify-sdk-python/issues/149), [#237](https://github.com/apify/apify-sdk-python/issues/237)
|
|
145
175
|
|
|
176
|
+
### Chore
|
|
177
|
+
|
|
178
|
+
- [**breaking**] Drop support for Python 3.8
|
|
179
|
+
|
|
146
180
|
|
|
147
181
|
## [1.7.2](https://github.com/apify/apify-sdk-python/releases/tag/v1.7.2) (2024-07-08)
|
|
148
182
|
|
|
183
|
+
- Add Actor Standby port
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
## [1.7.1](https://github.com/apify/apify-sdk-python/releases/tag/v1.7.1) (2024-05-23)
|
|
187
|
+
|
|
188
|
+
### 🐛 Bug Fixes
|
|
189
|
+
|
|
190
|
+
- Set a timeout for Actor cleanup
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
## [1.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.7.0) (2024-03-12)
|
|
194
|
+
|
|
149
195
|
### 🚀 Features
|
|
150
196
|
|
|
151
|
-
- Add
|
|
197
|
+
- Add a new way of generating the `uniqueKey` field of the request, aligning it with the Crawlee.
|
|
152
198
|
|
|
199
|
+
### 🐛 Bug Fixes
|
|
200
|
+
|
|
201
|
+
- Improve error handling for `to_apify_request` serialization failures
|
|
202
|
+
- Scrapy's `Request.dont_filter` works.
|
|
153
203
|
|
|
154
|
-
|
|
204
|
+
|
|
205
|
+
## [1.6.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.6.0) (2024-02-23)
|
|
155
206
|
|
|
156
207
|
### 🐛 Bug Fixes
|
|
157
208
|
|
|
158
|
-
-
|
|
209
|
+
- Update of Scrapy integration, fixes in `ApifyScheduler`, `to_apify_request` and `apply_apify_settings`.
|
|
159
210
|
|
|
211
|
+
### Chore
|
|
160
212
|
|
|
161
|
-
|
|
213
|
+
- Remove `ApifyRetryMiddleware` and stay with the Scrapy's default one
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
## [1.5.5](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.5) (2024-02-01)
|
|
217
|
+
|
|
218
|
+
### 🐛 Bug Fixes
|
|
219
|
+
|
|
220
|
+
- Fix conversion of `headers` fields in Apify <--> Scrapy request translation
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
## [1.5.4](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.4) (2024-01-24)
|
|
224
|
+
|
|
225
|
+
### 🐛 Bug Fixes
|
|
226
|
+
|
|
227
|
+
- Fix conversion of `userData` and `headers` fields in Apify <--> Scrapy request translation
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
## [1.5.3](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.3) (2024-01-23)
|
|
162
231
|
|
|
163
232
|
### 🚀 Features
|
|
164
233
|
|
|
165
|
-
-
|
|
234
|
+
- Add `apply_apify_settings` function to Scrapy subpackage
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
## [1.5.2](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.2) (2024-01-19)
|
|
238
|
+
|
|
239
|
+
### 🐛 Bug Fixes
|
|
240
|
+
|
|
241
|
+
- Add missing import check to `ApifyHttpProxyMiddleware`
|
|
242
|
+
|
|
243
|
+
### Chore
|
|
244
|
+
|
|
245
|
+
- Create a new subpackage for Scrapy pipelines
|
|
246
|
+
- Remove some noqas thanks to the new Ruff release
|
|
247
|
+
- Replace relative imports with absolute imports
|
|
248
|
+
- Replace asserts with custom checks in Scrapy subpackage
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
## [1.5.1](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.1) (2024-01-10)
|
|
252
|
+
|
|
253
|
+
### Chore
|
|
254
|
+
|
|
255
|
+
- Allowed running integration tests from PRs from forks, after maintainer approval
|
|
256
|
+
- Do not close `nested_event_loop` in the `Scheduler.__del__`
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
## [1.5.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.5.0) (2024-01-03)
|
|
260
|
+
|
|
261
|
+
### 🚀 Features
|
|
262
|
+
|
|
263
|
+
- Add `ApifyHttpProxyMiddleware`
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
## [1.4.1](https://github.com/apify/apify-sdk-python/releases/tag/v1.4.1) (2023-12-21)
|
|
267
|
+
|
|
268
|
+
### 🐛 Bug Fixes
|
|
269
|
+
|
|
270
|
+
- Resolve issue in `ApifyRetryMiddleware.process_exception()`, where requests were getting stuck in the request queue
|
|
271
|
+
|
|
272
|
+
### Chore
|
|
273
|
+
|
|
274
|
+
- Fix type hint problems for resource clients
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
## [1.4.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.4.0) (2023-12-05)
|
|
278
|
+
|
|
279
|
+
### Chore
|
|
280
|
+
|
|
281
|
+
- Migrate from Autopep8 and Flake8 to Ruff
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
## [1.3.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.3.0) (2023-11-15)
|
|
285
|
+
|
|
286
|
+
### 🚀 Features
|
|
287
|
+
|
|
288
|
+
- Add `scrapy` extra
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
## [1.2.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.2.0) (2023-10-23)
|
|
292
|
+
|
|
293
|
+
### 🚀 Features
|
|
294
|
+
|
|
295
|
+
- Add support for Python 3.12
|
|
296
|
+
|
|
297
|
+
### Chore
|
|
298
|
+
|
|
299
|
+
- Fix lint error (E721) in unit tests (for instance checks use `isinstance()`)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
## [1.1.5](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.5) (2023-10-03)
|
|
303
|
+
|
|
304
|
+
### 🚀 Features
|
|
305
|
+
|
|
306
|
+
- Update the Apify log formatter to contain an option for adding the logger name
|
|
307
|
+
|
|
308
|
+
### Chore
|
|
309
|
+
|
|
310
|
+
- Rewrite documentation publication to use Docusaurus
|
|
311
|
+
- Remove PR Toolkit workflow
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
## [1.1.4](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.4) (2023-09-06)
|
|
315
|
+
|
|
316
|
+
### 🐛 Bug Fixes
|
|
317
|
+
|
|
318
|
+
- Resolve issue with querying request queue head multiple times in parallel
|
|
319
|
+
|
|
320
|
+
### Chore
|
|
321
|
+
|
|
322
|
+
- Fix integration tests for Actor logger
|
|
323
|
+
- Remove `pytest-randomly` Pytest plugin
|
|
324
|
+
- Unpin `apify-client` and `apify-shared` to improve compatibility with their newer versions
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
## [1.1.3](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.3) (2023-08-25)
|
|
328
|
+
|
|
329
|
+
### Chore
|
|
330
|
+
|
|
331
|
+
- Unify indentation in configuration files
|
|
332
|
+
- Update the `Actor.reboot` method to use the new reboot endpoint
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
## [1.1.2](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.2) (2023-08-02)
|
|
336
|
+
|
|
337
|
+
### Chore
|
|
338
|
+
|
|
339
|
+
- Start importing general constants and utilities from the `apify-shared` library
|
|
340
|
+
- Simplify code via `flake8-simplify`
|
|
341
|
+
- Start using environment variables with prefix `ACTOR_` instead of some with prefix `APIFY_`
|
|
342
|
+
- Pin `apify-client` and `apify-shared` to prevent their implicit updates from breaking SDK
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
## [1.1.1](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.1) (2023-05-23)
|
|
346
|
+
|
|
347
|
+
### 🐛 Bug Fixes
|
|
348
|
+
|
|
349
|
+
- Relax dependency requirements to improve compatibility with other libraries
|
|
166
350
|
|
|
167
351
|
|
|
168
352
|
## [1.1.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.1.0) (2023-05-23)
|
|
169
353
|
|
|
170
354
|
### 🚀 Features
|
|
171
355
|
|
|
172
|
-
-
|
|
356
|
+
- Add option to add event handlers which accept no arguments
|
|
357
|
+
- Add support for `is_terminal` flag in status message update
|
|
358
|
+
- Add option to set status message along with `Actor.exit()`
|
|
359
|
+
|
|
360
|
+
### 🐛 Bug Fixes
|
|
361
|
+
|
|
362
|
+
- Start enforcing local storage to always use the UTF-8 encoding
|
|
363
|
+
- Fix saving key-value store values to local storage with the right extension for a given content type
|
|
364
|
+
|
|
365
|
+
### Chore
|
|
366
|
+
|
|
367
|
+
- Switch from `setup.py` to `pyproject.toml` for specifying project setup
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
## [1.0.0](https://github.com/apify/apify-sdk-python/releases/tag/v1.0.0) (2023-03-13)
|
|
371
|
+
|
|
372
|
+
### 🐛 Bug Fixes
|
|
373
|
+
|
|
374
|
+
- Fix `RequestQueue` not loading requests from an existing queue properly
|
|
375
|
+
|
|
376
|
+
### Chore
|
|
377
|
+
|
|
378
|
+
- Update to `apify-client` 1.0.0
|
|
379
|
+
- Start triggering base Docker image builds when releasing a new version
|
|
173
380
|
|
|
174
381
|
|
|
175
382
|
## [0.2.0](https://github.com/apify/apify-sdk-python/releases/tag/v0.2.0) (2023-03-06)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apify
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.1b5
|
|
4
4
|
Summary: Apify SDK for Python
|
|
5
5
|
Project-URL: Apify Homepage, https://apify.com
|
|
6
6
|
Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
|
|
@@ -225,8 +225,8 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
225
225
|
Classifier: Programming Language :: Python :: 3.13
|
|
226
226
|
Classifier: Topic :: Software Development :: Libraries
|
|
227
227
|
Requires-Python: >=3.10
|
|
228
|
-
Requires-Dist: apify-client
|
|
229
|
-
Requires-Dist: apify-shared
|
|
228
|
+
Requires-Dist: apify-client<2.0.0
|
|
229
|
+
Requires-Dist: apify-shared<2.0.0
|
|
230
230
|
Requires-Dist: crawlee~=0.6.0
|
|
231
231
|
Requires-Dist: cryptography>=42.0.0
|
|
232
232
|
Requires-Dist: httpx>=0.27.0
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "apify"
|
|
7
|
-
version = "2.7.
|
|
7
|
+
version = "2.7.1b5"
|
|
8
8
|
description = "Apify SDK for Python"
|
|
9
9
|
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -34,8 +34,8 @@ keywords = [
|
|
|
34
34
|
"scraping",
|
|
35
35
|
]
|
|
36
36
|
dependencies = [
|
|
37
|
-
"apify-client
|
|
38
|
-
"apify-shared
|
|
37
|
+
"apify-client<2.0.0",
|
|
38
|
+
"apify-shared<2.0.0",
|
|
39
39
|
"crawlee~=0.6.0",
|
|
40
40
|
"cryptography>=42.0.0",
|
|
41
41
|
"httpx>=0.27.0",
|
|
@@ -62,9 +62,10 @@ scrapy = ["scrapy>=2.11.0"]
|
|
|
62
62
|
|
|
63
63
|
[dependency-groups]
|
|
64
64
|
dev = [
|
|
65
|
-
"build~=1.
|
|
65
|
+
"build~=1.3.0",
|
|
66
|
+
"crawlee[parsel]",
|
|
66
67
|
"dycw-pytest-only>=2.1.1",
|
|
67
|
-
"griffe~=1.
|
|
68
|
+
"griffe~=1.9.0",
|
|
68
69
|
"mypy~=1.17.0",
|
|
69
70
|
"pre-commit~=4.2.0",
|
|
70
71
|
"pydoc-markdown~=4.8.0",
|
|
@@ -76,6 +77,7 @@ dev = [
|
|
|
76
77
|
"respx~=0.22.0",
|
|
77
78
|
"ruff~=0.12.0",
|
|
78
79
|
"setuptools", # setuptools are used by pytest but not explicitly required
|
|
80
|
+
"uvicorn[standard]",
|
|
79
81
|
]
|
|
80
82
|
|
|
81
83
|
[tool.hatch.build.targets.wheel]
|
|
@@ -126,3 +126,26 @@ async def test_something(
|
|
|
126
126
|
|
|
127
127
|
assert actor_run.status == 'SUCCEEDED'
|
|
128
128
|
```
|
|
129
|
+
|
|
130
|
+
### Asserts
|
|
131
|
+
|
|
132
|
+
Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown, without the left and right values. This means, we must include explicit assertion messages to aid potential debugging.
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
async def test_add_and_fetch_requests(
|
|
136
|
+
make_actor: MakeActorFunction,
|
|
137
|
+
run_actor: RunActorFunction,
|
|
138
|
+
) -> None:
|
|
139
|
+
"""Test basic functionality of adding and fetching requests."""
|
|
140
|
+
|
|
141
|
+
async def main() -> None:
|
|
142
|
+
async with Actor:
|
|
143
|
+
rq = await Actor.open_request_queue()
|
|
144
|
+
await rq.add_request(f'https://apify.com/')
|
|
145
|
+
assert is_finished is False, f'is_finished={is_finished}'
|
|
146
|
+
|
|
147
|
+
actor = await make_actor(label='rq-test', main_func=main)
|
|
148
|
+
run_result = await run_actor(actor)
|
|
149
|
+
|
|
150
|
+
assert run_result.status == 'SUCCEEDED'
|
|
151
|
+
```
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Test server is infinite server http://localhost:8080/{any_number} and each page has links to the next 10 pages.
|
|
3
|
+
For example:
|
|
4
|
+
http://localhost:8080/ contains links:
|
|
5
|
+
http://localhost:8080/0, http://localhost:8080/1, ..., http://localhost:8080/9
|
|
6
|
+
|
|
7
|
+
http://localhost:8080/1 contains links:
|
|
8
|
+
http://localhost:8080/10, http://localhost:8080/11, ..., http://localhost:8080/19
|
|
9
|
+
|
|
10
|
+
... and so on.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import asyncio
|
|
14
|
+
import logging
|
|
15
|
+
from collections.abc import Awaitable, Callable, Coroutine
|
|
16
|
+
from socket import socket
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from uvicorn import Config
|
|
20
|
+
from uvicorn.server import Server
|
|
21
|
+
from yarl import URL
|
|
22
|
+
|
|
23
|
+
Receive = Callable[[], Awaitable[dict[str, Any]]]
|
|
24
|
+
Send = Callable[[dict[str, Any]], Coroutine[None, None, None]]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def send_html_response(send: Send, html_content: bytes, status: int = 200) -> None:
|
|
28
|
+
"""Send an HTML response to the client."""
|
|
29
|
+
await send(
|
|
30
|
+
{
|
|
31
|
+
'type': 'http.response.start',
|
|
32
|
+
'status': status,
|
|
33
|
+
'headers': [[b'content-type', b'text/html; charset=utf-8']],
|
|
34
|
+
}
|
|
35
|
+
)
|
|
36
|
+
await send({'type': 'http.response.body', 'body': html_content})
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
async def app(scope: dict[str, Any], _: Receive, send: Send) -> None:
|
|
40
|
+
"""Main ASGI application handler that routes requests to specific handlers.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
scope: The ASGI connection scope.
|
|
44
|
+
_: The ASGI receive function.
|
|
45
|
+
send: The ASGI send function.
|
|
46
|
+
"""
|
|
47
|
+
assert scope['type'] == 'http'
|
|
48
|
+
path = scope['path']
|
|
49
|
+
|
|
50
|
+
links = '\n'.join(f'<a href="{path}{i}">{path}{i}</a>' for i in range(10))
|
|
51
|
+
await send_html_response(
|
|
52
|
+
send,
|
|
53
|
+
f"""\
|
|
54
|
+
<html><head>
|
|
55
|
+
<title>Title for {path} </title>
|
|
56
|
+
</head>
|
|
57
|
+
<body>
|
|
58
|
+
{links}
|
|
59
|
+
</body></html>""".encode(),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class TestServer(Server):
|
|
64
|
+
"""A test HTTP server implementation based on Uvicorn Server."""
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def url(self) -> URL:
|
|
68
|
+
"""Get the base URL of the server.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
A URL instance with the server's base URL.
|
|
72
|
+
"""
|
|
73
|
+
protocol = 'https' if self.config.is_ssl else 'http'
|
|
74
|
+
return URL(f'{protocol}://{self.config.host}:{self.config.port}/')
|
|
75
|
+
|
|
76
|
+
async def serve(self, sockets: list[socket] | None = None) -> None:
|
|
77
|
+
"""Run the server."""
|
|
78
|
+
if sockets:
|
|
79
|
+
raise RuntimeError('Simple TestServer does not support custom sockets')
|
|
80
|
+
self.restart_requested = asyncio.Event()
|
|
81
|
+
|
|
82
|
+
loop = asyncio.get_event_loop()
|
|
83
|
+
tasks = {
|
|
84
|
+
loop.create_task(super().serve()),
|
|
85
|
+
}
|
|
86
|
+
await asyncio.wait(tasks)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == '__main__':
|
|
90
|
+
asyncio.run(
|
|
91
|
+
TestServer(
|
|
92
|
+
config=Config(
|
|
93
|
+
app=app,
|
|
94
|
+
lifespan='off',
|
|
95
|
+
loop='asyncio',
|
|
96
|
+
port=8080,
|
|
97
|
+
log_config=None,
|
|
98
|
+
log_level=logging.CRITICAL,
|
|
99
|
+
)
|
|
100
|
+
).serve()
|
|
101
|
+
)
|
|
@@ -400,12 +400,12 @@ async def test_actor_adds_webhook_and_receives_event(
|
|
|
400
400
|
async with Actor:
|
|
401
401
|
|
|
402
402
|
class WebhookHandler(BaseHTTPRequestHandler):
|
|
403
|
-
def do_GET(self) -> None:
|
|
403
|
+
def do_GET(self) -> None:
|
|
404
404
|
self.send_response(200)
|
|
405
405
|
self.end_headers()
|
|
406
406
|
self.wfile.write(bytes('Hello, world!', encoding='utf-8'))
|
|
407
407
|
|
|
408
|
-
def do_POST(self) -> None:
|
|
408
|
+
def do_POST(self) -> None:
|
|
409
409
|
nonlocal webhook_body
|
|
410
410
|
content_length = self.headers.get('content-length')
|
|
411
411
|
length = int(content_length) if content_length else 0
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from .conftest import MakeActorFunction, RunActorFunction
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def test_actor_on_platform_max_crawl_depth(
|
|
10
|
+
make_actor: MakeActorFunction,
|
|
11
|
+
run_actor: RunActorFunction,
|
|
12
|
+
) -> None:
|
|
13
|
+
"""Test that the actor respects max_crawl_depth."""
|
|
14
|
+
|
|
15
|
+
async def main() -> None:
|
|
16
|
+
"""The crawler entry point."""
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
|
20
|
+
|
|
21
|
+
from apify import Actor
|
|
22
|
+
|
|
23
|
+
async with Actor:
|
|
24
|
+
crawler = ParselCrawler(max_crawl_depth=2)
|
|
25
|
+
finished = []
|
|
26
|
+
enqueue_pattern = re.compile(r'http://localhost:8080/2+$')
|
|
27
|
+
|
|
28
|
+
@crawler.router.default_handler
|
|
29
|
+
async def default_handler(context: ParselCrawlingContext) -> None:
|
|
30
|
+
"""Default request handler."""
|
|
31
|
+
context.log.info(f'Processing {context.request.url} ...')
|
|
32
|
+
await context.enqueue_links(include=[enqueue_pattern])
|
|
33
|
+
finished.append(context.request.url)
|
|
34
|
+
|
|
35
|
+
await crawler.run(['http://localhost:8080/'])
|
|
36
|
+
assert finished == ['http://localhost:8080/', 'http://localhost:8080/2', 'http://localhost:8080/22']
|
|
37
|
+
|
|
38
|
+
actor = await make_actor(label='crawler-max-depth', main_func=main)
|
|
39
|
+
run_result = await run_actor(actor)
|
|
40
|
+
|
|
41
|
+
assert run_result.status == 'SUCCEEDED'
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def test_actor_on_platform_max_requests_per_crawl(
|
|
45
|
+
make_actor: MakeActorFunction,
|
|
46
|
+
run_actor: RunActorFunction,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Test that the actor respects max_requests_per_crawl."""
|
|
49
|
+
|
|
50
|
+
async def main() -> None:
|
|
51
|
+
"""The crawler entry point."""
|
|
52
|
+
from crawlee import ConcurrencySettings
|
|
53
|
+
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
|
54
|
+
|
|
55
|
+
from apify import Actor
|
|
56
|
+
|
|
57
|
+
async with Actor:
|
|
58
|
+
crawler = ParselCrawler(
|
|
59
|
+
max_requests_per_crawl=3, concurrency_settings=ConcurrencySettings(max_concurrency=1)
|
|
60
|
+
)
|
|
61
|
+
finished = []
|
|
62
|
+
|
|
63
|
+
@crawler.router.default_handler
|
|
64
|
+
async def default_handler(context: ParselCrawlingContext) -> None:
|
|
65
|
+
"""Default request handler."""
|
|
66
|
+
context.log.info(f'Processing {context.request.url} ...')
|
|
67
|
+
await context.enqueue_links()
|
|
68
|
+
finished.append(context.request.url)
|
|
69
|
+
|
|
70
|
+
await crawler.run(['http://localhost:8080/'])
|
|
71
|
+
assert len(finished) == 3
|
|
72
|
+
|
|
73
|
+
actor = await make_actor(label='crawler-max-requests', main_func=main)
|
|
74
|
+
run_result = await run_actor(actor)
|
|
75
|
+
|
|
76
|
+
assert run_result.status == 'SUCCEEDED'
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def test_actor_on_platform_max_request_retries(
|
|
80
|
+
make_actor: MakeActorFunction,
|
|
81
|
+
run_actor: RunActorFunction,
|
|
82
|
+
) -> None:
|
|
83
|
+
"""Test that the actor respects max_request_retries."""
|
|
84
|
+
|
|
85
|
+
async def main() -> None:
|
|
86
|
+
"""The crawler entry point."""
|
|
87
|
+
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
|
|
88
|
+
|
|
89
|
+
from apify import Actor
|
|
90
|
+
|
|
91
|
+
async with Actor:
|
|
92
|
+
max_retries = 3
|
|
93
|
+
crawler = ParselCrawler(max_request_retries=max_retries)
|
|
94
|
+
failed_counter = 0
|
|
95
|
+
|
|
96
|
+
@crawler.error_handler
|
|
97
|
+
async def error_handler(_: BasicCrawlingContext, __: Exception) -> None:
|
|
98
|
+
nonlocal failed_counter
|
|
99
|
+
failed_counter += 1
|
|
100
|
+
|
|
101
|
+
@crawler.router.default_handler
|
|
102
|
+
async def default_handler(_: ParselCrawlingContext) -> None:
|
|
103
|
+
raise RuntimeError('Some error')
|
|
104
|
+
|
|
105
|
+
await crawler.run(['http://localhost:8080/'])
|
|
106
|
+
assert failed_counter == max_retries, f'{failed_counter=}'
|
|
107
|
+
|
|
108
|
+
actor = await make_actor(label='crawler-max-retries', main_func=main)
|
|
109
|
+
run_result = await run_actor(actor)
|
|
110
|
+
|
|
111
|
+
assert run_result.status == 'SUCCEEDED'
|