apify 2.7.1b19__tar.gz → 2.7.1b21__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- {apify-2.7.1b19 → apify-2.7.1b21}/CHANGELOG.md +26 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/PKG-INFO +2 -2
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/12_pay_per_event.mdx +6 -0
- apify-2.7.1b21/docs/04_upgrading/upgrading_to_v3.md +159 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/pyproject.toml +2 -2
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_actor.py +37 -69
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_configuration.py +6 -6
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/__init__.py +2 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_dataset_client.py +9 -2
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_key_value_store_client.py +9 -2
- apify-2.7.1b21/src/apify/storage_clients/_apify/_request_queue_client.py +327 -0
- apify-2.7.1b19/src/apify/storage_clients/_apify/_request_queue_client.py → apify-2.7.1b21/src/apify/storage_clients/_apify/_request_queue_shared_client.py +36 -316
- apify-2.7.1b21/src/apify/storage_clients/_apify/_request_queue_single_client.py +399 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_storage_client.py +26 -4
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_utils.py +30 -3
- apify-2.7.1b21/src/apify/storage_clients/_smart_apify/__init__.py +1 -0
- apify-2.7.1b21/src/apify/storage_clients/_smart_apify/_storage_client.py +117 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/requirements.txt +1 -1
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/conftest.py +10 -5
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_request_queue.py +7 -4
- apify-2.7.1b21/tests/integration/test_apify_storages.py +152 -0
- apify-2.7.1b21/tests/integration/test_request_queue.py +1191 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_log.py +35 -31
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_configuration.py +3 -17
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/storage_clients/test_apify_request_queue_client.py +1 -1
- {apify-2.7.1b19 → apify-2.7.1b21}/uv.lock +466 -380
- {apify-2.7.1b19 → apify-2.7.1b21}/website/package-lock.json +809 -707
- apify-2.7.1b19/docs/04_upgrading/upgrading_to_v3.md +0 -54
- apify-2.7.1b19/tests/integration/test_apify_storages.py +0 -34
- apify-2.7.1b19/tests/integration/test_request_queue.py +0 -1304
- {apify-2.7.1b19 → apify-2.7.1b21}/.editorconfig +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/CODEOWNERS +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/build_and_deploy_docs.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/check_pr_title.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/pre_release.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/release.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/run_code_checks.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.github/workflows/update_new_issue.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.gitignore +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.markdownlint.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/.pre-commit-config.yaml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/CONTRIBUTING.md +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/LICENSE +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/Makefile +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/README.md +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/01_introduction.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/02_running_actors_locally.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/03_actor_structure.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/code/01_introduction.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/code/actor_structure/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/code/actor_structure/__main__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/code/actor_structure/main.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/01_overview/code/actor_structure/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/01_actor_lifecycle.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/02_actor_input.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/03_storages.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/04_actor_events.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/05_proxy_management.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/06_interacting_with_other_actors.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/07_webhooks.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/08_access_apify_api.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/09_running_webserver.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/10_logging.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/11_configuration.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/01_context_manager.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/01_init_exit.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/01_reboot.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/01_status_message.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/02_input.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_dataset_exports.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_dataset_read_write.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_deleting_storages.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_kvs_iterating.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_kvs_public_url.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_kvs_read_write.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_opening_storages.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/03_rq.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/04_actor_events.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_apify_proxy.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_apify_proxy_config.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_custom_proxy.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_custom_proxy_function.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_proxy_actor_input.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_proxy_httpx.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/05_proxy_rotation.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/06_interacting_call.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/06_interacting_call_task.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/06_interacting_metamorph.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/06_interacting_start.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/07_webhook.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/07_webhook_preventing.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/08_actor_client.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/08_actor_new_client.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/09_webserver.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/10_log_config.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/10_logger_usage.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/10_redirect_log.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/10_redirect_log_existing_run.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/11_config.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/actor_charge.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/02_concepts/code/conditional_actor_charge.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/01_beautifulsoup_httpx.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/02_parsel_impit.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/03_playwright.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/04_selenium.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/05_crawlee.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/06_scrapy.mdx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/01_beautifulsoup_httpx.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/02_parsel_impit.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/03_playwright.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/04_selenium.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/05_crawlee_beautifulsoup.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/05_crawlee_parsel.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/05_crawlee_playwright.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/__main__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/items.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/main.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/settings.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/spiders/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/spiders/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/03_guides/code/scrapy_project/src/spiders/title.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/04_upgrading/upgrading_to_v2.md +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/docs/pyproject.toml +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/renovate.json +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_charging.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_consts.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_crypto.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_models.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_proxy_configuration.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/_utils.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/events/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/events/_apify_event_manager.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/events/_types.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/events/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/log.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/request_loaders/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/request_loaders/_apify_request_list.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/request_loaders/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/_actor_runner.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/_async_thread.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/_logging_config.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/extensions/_httpcache.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/middlewares/apify_proxy.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/middlewares/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/pipelines/actor_dataset_push.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/pipelines/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/requests.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/scheduler.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/scrapy/utils.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_models.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_file_system/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_file_system/_key_value_store_client.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_file_system/_storage_client.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storages/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storages/py.typed +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/README.md +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/_utils.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/Dockerfile +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/server.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/src/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/src/__main__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/actor_source_base/src/main.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_api_helpers.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_call_timeouts.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_charge.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_dataset.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_events.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_key_value_store.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_log.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_actor_scrapy.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_crawlers_with_storages.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/integration/test_fixtures.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_dataset.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_env_helpers.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_helpers.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_key_value_store.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_non_default_instance.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_actor_request_queue.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/actor/test_request_list.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/conftest.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/events/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/events/test_apify_event_manager.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/extensions/test_httpcache.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/middlewares/test_apify_proxy.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/pipelines/test_actor_dataset_push.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/requests/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/requests/test_to_apify_request.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/requests/test_to_scrapy_request.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/utils/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/utils/test_apply_apify_settings.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/scrapy/utils/test_get_basic_auth_header.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/storage_clients/__init__.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/storage_clients/test_file_system.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/test_apify_storages.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/test_crypto.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/tests/unit/test_proxy_configuration.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/.eslintrc.json +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/babel.config.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/build_api_reference.sh +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/docusaurus.config.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/generate_module_shortcuts.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/package.json +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/sidebars.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/ApiLink.jsx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/Gradients.jsx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/Highlights.jsx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/Highlights.module.css +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/RunnableCodeBlock.jsx +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/components/RunnableCodeBlock.module.css +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/css/custom.css +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/pages/home_page_example.py +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/pages/index.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/pages/index.module.css +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/src/theme/DocItem/Content/index.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/static/.nojekyll +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/static/img/docs-og.png +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/static/img/guides/redirected_logs_example.webp +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/tools/docs-prettier.config.js +0 -0
- {apify-2.7.1b19 → apify-2.7.1b21}/website/tools/utils/externalLink.js +0 -0
|
@@ -11,6 +11,7 @@ All notable changes to this project will be documented in this file.
|
|
|
11
11
|
- Add new methods to ChargingManager ([#580](https://github.com/apify/apify-sdk-python/pull/580)) ([54f7f8b](https://github.com/apify/apify-sdk-python/commit/54f7f8b29c5982be98b595dac11eceff915035c9)) by [@vdusek](https://github.com/vdusek)
|
|
12
12
|
- Add support for NDU storages ([#594](https://github.com/apify/apify-sdk-python/pull/594)) ([8721ef5](https://github.com/apify/apify-sdk-python/commit/8721ef5731bcb1a04ad63c930089bf83be29f308)) by [@vdusek](https://github.com/vdusek), closes [#1175](https://github.com/apify/apify-sdk-python/issues/1175)
|
|
13
13
|
- Add stats to `ApifyRequestQueueClient` ([#574](https://github.com/apify/apify-sdk-python/pull/574)) ([21f6782](https://github.com/apify/apify-sdk-python/commit/21f6782b444f623aba986b4922cf67bafafd4b2c)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1344](https://github.com/apify/apify-sdk-python/issues/1344)
|
|
14
|
+
- Add specialized ApifyRequestQueue clients ([#573](https://github.com/apify/apify-sdk-python/pull/573)) ([f830ab0](https://github.com/apify/apify-sdk-python/commit/f830ab09b1fa12189c9d3297d5cf18a4f2da62fa)) by [@Pijukatel](https://github.com/Pijukatel)
|
|
14
15
|
|
|
15
16
|
### 🐛 Bug Fixes
|
|
16
17
|
|
|
@@ -30,6 +31,7 @@ All notable changes to this project will be documented in this file.
|
|
|
30
31
|
- [**breaking**] Replace `httpx` with `impit` ([#560](https://github.com/apify/apify-sdk-python/pull/560)) ([cca3869](https://github.com/apify/apify-sdk-python/commit/cca3869e85968865e56aafcdcb36fbccba27aef0)) by [@Mantisus](https://github.com/Mantisus), closes [#558](https://github.com/apify/apify-sdk-python/issues/558)
|
|
31
32
|
- [**breaking**] Remove `Request.id` field ([#553](https://github.com/apify/apify-sdk-python/pull/553)) ([445ab5d](https://github.com/apify/apify-sdk-python/commit/445ab5d752b785fc2018b35c8adbe779253d7acd)) by [@Pijukatel](https://github.com/Pijukatel)
|
|
32
33
|
- [**breaking**] Make `Actor` initialization stricter and more predictable ([#576](https://github.com/apify/apify-sdk-python/pull/576)) ([912222a](https://github.com/apify/apify-sdk-python/commit/912222a7a8123be66c94c50a2e461276fbfc50c4)) by [@Pijukatel](https://github.com/Pijukatel)
|
|
34
|
+
- [**breaking**] Make default Apify storages use alias mechanism ([#606](https://github.com/apify/apify-sdk-python/pull/606)) ([dbea7d9](https://github.com/apify/apify-sdk-python/commit/dbea7d97fe7f25aa8658a32c5bb46a3800561df5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#599](https://github.com/apify/apify-sdk-python/issues/599)
|
|
33
35
|
|
|
34
36
|
|
|
35
37
|
<!-- git-cliff-unreleased-end -->
|
|
@@ -165,6 +167,30 @@ All notable changes to this project will be documented in this file.
|
|
|
165
167
|
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
166
168
|
|
|
167
169
|
|
|
170
|
+
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
171
|
+
|
|
172
|
+
### 🚀 Features
|
|
173
|
+
|
|
174
|
+
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
175
|
+
|
|
176
|
+
### 🐛 Bug Fixes
|
|
177
|
+
|
|
178
|
+
- Sync `@docusaurus` theme version [internal] ([#500](https://github.com/apify/apify-sdk-python/pull/500)) ([a7485e7](https://github.com/apify/apify-sdk-python/commit/a7485e7d2276fde464ce862573d5b95e7d4d836a)) by [@katzino](https://github.com/katzino)
|
|
179
|
+
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
183
|
+
|
|
184
|
+
### 🚀 Features
|
|
185
|
+
|
|
186
|
+
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
187
|
+
|
|
188
|
+
### 🐛 Bug Fixes
|
|
189
|
+
|
|
190
|
+
- Sync `@docusaurus` theme version [internal] ([#500](https://github.com/apify/apify-sdk-python/pull/500)) ([a7485e7](https://github.com/apify/apify-sdk-python/commit/a7485e7d2276fde464ce862573d5b95e7d4d836a)) by [@katzino](https://github.com/katzino)
|
|
191
|
+
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
192
|
+
|
|
193
|
+
|
|
168
194
|
|
|
169
195
|
## [2.7.3](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.3) (2025-08-11)
|
|
170
196
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apify
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.1b21
|
|
4
4
|
Summary: Apify SDK for Python
|
|
5
5
|
Project-URL: Apify Homepage, https://apify.com
|
|
6
6
|
Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
|
|
@@ -228,7 +228,7 @@ Requires-Python: >=3.10
|
|
|
228
228
|
Requires-Dist: apify-client<3.0.0,>=2.0.0
|
|
229
229
|
Requires-Dist: apify-shared<3.0.0,>=2.0.0
|
|
230
230
|
Requires-Dist: cachetools>=5.5.0
|
|
231
|
-
Requires-Dist: crawlee==0.6.
|
|
231
|
+
Requires-Dist: crawlee==0.6.13b46
|
|
232
232
|
Requires-Dist: cryptography>=42.0.0
|
|
233
233
|
Requires-Dist: impit>=0.6.1
|
|
234
234
|
Requires-Dist: lazy-object-proxy>=1.11.0
|
|
@@ -13,6 +13,12 @@ Apify provides several [pricing models](https://docs.apify.com/platform/actors/p
|
|
|
13
13
|
|
|
14
14
|
To use the pay-per-event pricing model, you first need to [set it up](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event) for your Actor in the Apify console. After that, you're free to start charging for events.
|
|
15
15
|
|
|
16
|
+
:::info How pay-per-event pricing works
|
|
17
|
+
|
|
18
|
+
If you want more details about PPE pricing, please refer to our [PPE documentation](https://docs.apify.com/platform/actors/publishing/monetize/pay-per-event).
|
|
19
|
+
|
|
20
|
+
:::
|
|
21
|
+
|
|
16
22
|
## Charging for events
|
|
17
23
|
|
|
18
24
|
After monetization is set in the Apify console, you can add <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> calls to your code and start monetizing!
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: upgrading-to-v3
|
|
3
|
+
title: Upgrading to v3
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
This page summarizes the breaking changes between Apify Python SDK v2.x and v3.0.
|
|
7
|
+
|
|
8
|
+
## Python version support
|
|
9
|
+
|
|
10
|
+
Support for Python 3.9 has been dropped. The Apify Python SDK v3.x now requires Python 3.10 or later. Make sure your environment is running a compatible version before upgrading.
|
|
11
|
+
|
|
12
|
+
## Changes in storages
|
|
13
|
+
|
|
14
|
+
Apify Python SDK v3.0 includes Crawlee v1.0, which brings significant changes to the storage APIs. In Crawlee v1.0, the `Dataset`, `KeyValueStore`, and `RequestQueue` storage APIs have been updated for consistency and simplicity. Below is a detailed overview of what's new, what's changed, and what's been removed.
|
|
15
|
+
|
|
16
|
+
See the Crawlee's [Storages guide](https://crawlee.dev/python/docs/guides/storages) for more details.
|
|
17
|
+
|
|
18
|
+
### Dataset
|
|
19
|
+
|
|
20
|
+
The `Dataset` API now includes several new methods, such as:
|
|
21
|
+
|
|
22
|
+
- `get_metadata` - retrieves metadata information for the dataset.
|
|
23
|
+
- `purge` - completely clears the dataset, including all items (keeps the metadata only).
|
|
24
|
+
- `list_items` - returns the dataset's items in a list format.
|
|
25
|
+
|
|
26
|
+
Some older methods have been removed or replaced:
|
|
27
|
+
|
|
28
|
+
- `from_storage_object` constructor has been removed. You should now use the `open` method with either a `name` or `id` parameter.
|
|
29
|
+
- `get_info` method and the `storage_object` property have been replaced by the new `get_metadata` method.
|
|
30
|
+
- `set_metadata` method has been removed.
|
|
31
|
+
- `write_to_json` and `write_to_csv` methods have been removed; instead, use the `export_to` method for exporting data in different formats.
|
|
32
|
+
|
|
33
|
+
### Key-value store
|
|
34
|
+
|
|
35
|
+
The `KeyValueStore` API now includes several new methods, such as:
|
|
36
|
+
|
|
37
|
+
- `get_metadata` - retrieves metadata information for the key-value store.
|
|
38
|
+
- `purge` - completely clears the key-value store, removing all keys and values (keeps the metadata only).
|
|
39
|
+
- `delete_value` - deletes a specific key and its associated value.
|
|
40
|
+
- `list_keys` - lists all keys in the key-value store.
|
|
41
|
+
|
|
42
|
+
Some older methods have been removed or replaced:
|
|
43
|
+
|
|
44
|
+
- `from_storage_object` - removed; use the `open` method with either a `name` or `id` instead.
|
|
45
|
+
- `get_info` and `storage_object` - replaced by the new `get_metadata` method.
|
|
46
|
+
- `set_metadata` method has been removed.
|
|
47
|
+
|
|
48
|
+
### Request queue
|
|
49
|
+
|
|
50
|
+
The `RequestQueue` API now includes several new methods, such as:
|
|
51
|
+
|
|
52
|
+
- `get_metadata` - retrieves metadata information for the request queue.
|
|
53
|
+
- `purge` - completely clears the request queue, including all pending and processed requests (keeps the metadata only).
|
|
54
|
+
- `add_requests` - replaces the previous `add_requests_batched` method, offering the same functionality under a simpler name.
|
|
55
|
+
|
|
56
|
+
Some older methods have been removed or replaced:
|
|
57
|
+
|
|
58
|
+
- `from_storage_object` - removed; use the `open` method with either a `name` or `id` instead.
|
|
59
|
+
- `get_info` and `storage_object` - replaced by the new `get_metadata` method.
|
|
60
|
+
- `get_request` has argument `unique_key` instead of `request_id` as the `id` field was removed from the `Request`.
|
|
61
|
+
- `set_metadata` method has been removed.
|
|
62
|
+
|
|
63
|
+
Some changes in the related model classes:
|
|
64
|
+
|
|
65
|
+
- `resource_directory` in `RequestQueueMetadata` - removed; use the corresponding `path_to_*` property instead.
|
|
66
|
+
- `stats` field in `RequestQueueMetadata` - removed as it was unused.
|
|
67
|
+
- `RequestQueueHead` - replaced by `RequestQueueHeadWithLocks`.
|
|
68
|
+
|
|
69
|
+
## Removed Actor.config property
|
|
70
|
+
- `Actor.config` property has been removed. Use `Actor.configuration` instead.
|
|
71
|
+
|
|
72
|
+
## Default storage ids in configuration changed to None
|
|
73
|
+
- `Configuration.default_key_value_store_id` changed from `'default'` to `None`.
|
|
74
|
+
- `Configuration.default_dataset_id` changed from `'default'` to `None`.
|
|
75
|
+
- `Configuration.default_request_queue_id` changed from `'default'` to `None`.
|
|
76
|
+
|
|
77
|
+
Previously using the default storage without specifying its `id` in `Configuration` would lead to using specific storage with id `'default'`. Now it will use newly created unnamed storage with `'id'` assigned by the Apify platform, consecutive calls to get the default storage will return the same storage.
|
|
78
|
+
|
|
79
|
+
## Actor initialization and ServiceLocator changes
|
|
80
|
+
|
|
81
|
+
`Actor` initialization and global `service_locator` services setup is more strict and predictable.
|
|
82
|
+
- Services in `Actor` can't be changed after calling `Actor.init`, entering the `async with Actor` context manager or after requesting them from the `Actor`.
|
|
83
|
+
- Services in `Actor` can be different from services in Crawler.
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
**Now (v3.0):**
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from crawlee.crawlers import BasicCrawler
|
|
90
|
+
from crawlee.storage_clients import MemoryStorageClient
|
|
91
|
+
from crawlee.configuration import Configuration
|
|
92
|
+
from crawlee.events import LocalEventManager
|
|
93
|
+
from apify import Actor
|
|
94
|
+
|
|
95
|
+
async def main():
|
|
96
|
+
|
|
97
|
+
async with Actor():
|
|
98
|
+
# This crawler will use same services as Actor and global service_locator
|
|
99
|
+
crawler_1 = BasicCrawler()
|
|
100
|
+
|
|
101
|
+
# This crawler will use custom services
|
|
102
|
+
custom_configuration = Configuration()
|
|
103
|
+
custom_event_manager = LocalEventManager.from_config(custom_configuration)
|
|
104
|
+
custom_storage_client = MemoryStorageClient()
|
|
105
|
+
crawler_2 = BasicCrawler(
|
|
106
|
+
configuration=custom_configuration,
|
|
107
|
+
event_manager=custom_event_manager,
|
|
108
|
+
storage_client=custom_storage_client,
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### Changes in storage clients
|
|
113
|
+
|
|
114
|
+
## Explicit control over storage clients used in Actor
|
|
115
|
+
- It is now possible to have full control over which storage clients are used by the `Actor`. To make development of Actors convenient, the `Actor` has two storage clients. One that is used when running on Apify platform or when opening storages with `force_cloud=True` and the other client that is used when running outside the Apify platform. The `Actor` has reasonable defaults and for the majority of use-cases there is no need to change it. However, if you need to use a different storage client, you can set it up before entering `Actor` context through `service_locator`.
|
|
116
|
+
|
|
117
|
+
**Now (v3.0):**
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from crawlee import service_locator
|
|
121
|
+
from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient, MemoryStorageClient
|
|
122
|
+
from apify import Actor
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
async def main():
|
|
126
|
+
service_locator.set_storage_client(
|
|
127
|
+
SmartApifyStorageClient(
|
|
128
|
+
cloud_storage_client=ApifyStorageClient(request_queue_access="single"),
|
|
129
|
+
local_storage_client=MemoryStorageClient()
|
|
130
|
+
)
|
|
131
|
+
)
|
|
132
|
+
async with Actor:
|
|
133
|
+
rq = await Actor.open_request_queue()
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
## The default use of optimized ApifyRequestQueueClient
|
|
138
|
+
|
|
139
|
+
- The default client for working with Apify platform based `RequestQueue` is now optimized and simplified client which does significantly lower amount of API calls, but does not support multiple consumers working on the same queue. It is cheaper and faster and is suitable for the majority of the use cases.
|
|
140
|
+
- The full client is still available, but it has to be explicitly requested via `request_queue_access="shared"` argument when using the `ApifyStorageClient`.
|
|
141
|
+
|
|
142
|
+
**Now (v3.0):**
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from crawlee import service_locator
|
|
146
|
+
from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient
|
|
147
|
+
from apify import Actor
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
async def main():
|
|
151
|
+
# Full client that supports multiple consumers of the Apify Request Queue
|
|
152
|
+
service_locator.set_storage_client(
|
|
153
|
+
SmartApifyStorageClient(
|
|
154
|
+
cloud_storage_client=ApifyStorageClient(request_queue_access="shared"),
|
|
155
|
+
)
|
|
156
|
+
)
|
|
157
|
+
async with Actor:
|
|
158
|
+
rq = await Actor.open_request_queue()
|
|
159
|
+
```
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "apify"
|
|
7
|
-
version = "2.7.
|
|
7
|
+
version = "2.7.1b21"
|
|
8
8
|
description = "Apify SDK for Python"
|
|
9
9
|
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -36,7 +36,7 @@ keywords = [
|
|
|
36
36
|
dependencies = [
|
|
37
37
|
"apify-client>=2.0.0,<3.0.0",
|
|
38
38
|
"apify-shared>=2.0.0,<3.0.0",
|
|
39
|
-
"crawlee==0.6.
|
|
39
|
+
"crawlee==0.6.13b46",
|
|
40
40
|
"cachetools>=5.5.0",
|
|
41
41
|
"cryptography>=42.0.0",
|
|
42
42
|
"impit>=0.6.1",
|
|
@@ -25,7 +25,6 @@ from crawlee.events import (
|
|
|
25
25
|
EventPersistStateData,
|
|
26
26
|
EventSystemInfoData,
|
|
27
27
|
)
|
|
28
|
-
from crawlee.storage_clients import FileSystemStorageClient
|
|
29
28
|
|
|
30
29
|
from apify._charging import ChargeResult, ChargingManager, ChargingManagerImplementation
|
|
31
30
|
from apify._configuration import Configuration
|
|
@@ -38,6 +37,7 @@ from apify.events import ApifyEventManager, EventManager, LocalEventManager
|
|
|
38
37
|
from apify.log import _configure_logging, logger
|
|
39
38
|
from apify.storage_clients import ApifyStorageClient
|
|
40
39
|
from apify.storage_clients._file_system import ApifyFileSystemStorageClient
|
|
40
|
+
from apify.storage_clients._smart_apify._storage_client import SmartApifyStorageClient
|
|
41
41
|
from apify.storages import Dataset, KeyValueStore, RequestQueue
|
|
42
42
|
|
|
43
43
|
if TYPE_CHECKING:
|
|
@@ -48,7 +48,6 @@ if TYPE_CHECKING:
|
|
|
48
48
|
from typing_extensions import Self
|
|
49
49
|
|
|
50
50
|
from crawlee.proxy_configuration import _NewUrlFunction
|
|
51
|
-
from crawlee.storage_clients import StorageClient
|
|
52
51
|
|
|
53
52
|
from apify._models import Webhook
|
|
54
53
|
|
|
@@ -131,7 +130,6 @@ class _ActorType:
|
|
|
131
130
|
self._configuration = configuration
|
|
132
131
|
self._configure_logging = configure_logging
|
|
133
132
|
self._apify_client: ApifyClientAsync | None = None
|
|
134
|
-
self._local_storage_client: StorageClient | None = None
|
|
135
133
|
|
|
136
134
|
self._is_initialized = False
|
|
137
135
|
|
|
@@ -234,45 +232,42 @@ class _ActorType:
|
|
|
234
232
|
"""The logging.Logger instance the Actor uses."""
|
|
235
233
|
return logger
|
|
236
234
|
|
|
237
|
-
def
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
235
|
+
def _raise_if_not_initialized(self) -> None:
|
|
236
|
+
if not self._is_initialized:
|
|
237
|
+
raise RuntimeError('The Actor was not initialized!')
|
|
238
|
+
|
|
239
|
+
@cached_property
|
|
240
|
+
def _storage_client(self) -> SmartApifyStorageClient:
|
|
241
|
+
"""Storage client used by the actor.
|
|
241
242
|
|
|
243
|
+
Depending on the initialization of the service locator the client can be created in different ways.
|
|
244
|
+
"""
|
|
242
245
|
try:
|
|
243
|
-
#
|
|
244
|
-
implicit_storage_client =
|
|
246
|
+
# Nothing was set by the user.
|
|
247
|
+
implicit_storage_client = SmartApifyStorageClient(
|
|
248
|
+
local_storage_client=ApifyFileSystemStorageClient(), cloud_storage_client=ApifyStorageClient()
|
|
249
|
+
)
|
|
245
250
|
service_locator.set_storage_client(implicit_storage_client)
|
|
246
|
-
self._local_storage_client = implicit_storage_client
|
|
247
251
|
except ServiceConflictError:
|
|
248
252
|
self.log.debug(
|
|
249
253
|
'Storage client in service locator was set explicitly before Actor.init was called.'
|
|
250
254
|
'Using the existing storage client as implicit storage client for the Actor.'
|
|
251
255
|
)
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
def _raise_if_cloud_requested_but_not_configured(self, *, force_cloud: bool) -> None:
|
|
268
|
-
if not force_cloud:
|
|
269
|
-
return
|
|
270
|
-
|
|
271
|
-
if not self.is_at_home() and self.configuration.token is None:
|
|
272
|
-
raise RuntimeError(
|
|
273
|
-
'In order to use the Apify cloud storage from your computer, '
|
|
274
|
-
'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
|
|
275
|
-
)
|
|
256
|
+
else:
|
|
257
|
+
return implicit_storage_client
|
|
258
|
+
|
|
259
|
+
# User set something in the service locator.
|
|
260
|
+
explicit_storage_client = service_locator.get_storage_client()
|
|
261
|
+
if isinstance(explicit_storage_client, SmartApifyStorageClient):
|
|
262
|
+
# The client was manually set to the right type in the service locator. This is the explicit way.
|
|
263
|
+
return explicit_storage_client
|
|
264
|
+
|
|
265
|
+
raise RuntimeError(
|
|
266
|
+
'The storage client in the service locator has to be instance of SmartApifyStorageClient. If you want to '
|
|
267
|
+
'set the storage client manually you have to call '
|
|
268
|
+
'`service_locator.set_storage_client(SmartApifyStorageClient(...))` before entering Actor context or '
|
|
269
|
+
'awaiting `Actor.init`.'
|
|
270
|
+
)
|
|
276
271
|
|
|
277
272
|
async def init(self) -> None:
|
|
278
273
|
"""Initialize the Actor instance.
|
|
@@ -285,6 +280,7 @@ class _ActorType:
|
|
|
285
280
|
This method should be called immediately before performing any additional Actor actions, and it should be
|
|
286
281
|
called only once.
|
|
287
282
|
"""
|
|
283
|
+
self.log.info('Initializing Actor...')
|
|
288
284
|
if self._configuration:
|
|
289
285
|
# Set explicitly the configuration in the service locator
|
|
290
286
|
service_locator.set_configuration(self.configuration)
|
|
@@ -298,22 +294,13 @@ class _ActorType:
|
|
|
298
294
|
if _ActorType._is_any_instance_initialized:
|
|
299
295
|
self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care')
|
|
300
296
|
|
|
301
|
-
# Create an instance of the cloud storage client, the local storage client is obtained
|
|
302
|
-
# from the service locator
|
|
303
|
-
self._cloud_storage_client = ApifyStorageClient()
|
|
304
|
-
|
|
305
297
|
# Make sure that the currently initialized instance is also available through the global `Actor` proxy
|
|
306
298
|
cast('Proxy', Actor).__wrapped__ = self
|
|
307
299
|
|
|
308
300
|
self._is_exiting = False
|
|
309
301
|
self._was_final_persist_state_emitted = False
|
|
310
302
|
|
|
311
|
-
|
|
312
|
-
if self.is_at_home():
|
|
313
|
-
service_locator.set_storage_client(self._cloud_storage_client)
|
|
314
|
-
self._local_storage_client = self._cloud_storage_client
|
|
315
|
-
else:
|
|
316
|
-
self._get_local_storage_client()
|
|
303
|
+
self.log.debug(f'Storage client set to {self._storage_client}')
|
|
317
304
|
|
|
318
305
|
service_locator.set_event_manager(self.event_manager)
|
|
319
306
|
|
|
@@ -321,12 +308,8 @@ class _ActorType:
|
|
|
321
308
|
if self._configure_logging:
|
|
322
309
|
_configure_logging()
|
|
323
310
|
|
|
324
|
-
self.log.info('Initializing Actor...')
|
|
325
311
|
self.log.info('System info', extra=get_system_info())
|
|
326
312
|
|
|
327
|
-
# TODO: Print outdated SDK version warning (we need a new env var for this)
|
|
328
|
-
# https://github.com/apify/apify-sdk-python/issues/146
|
|
329
|
-
|
|
330
313
|
await self.event_manager.__aenter__()
|
|
331
314
|
self.log.debug('Event manager initialized')
|
|
332
315
|
|
|
@@ -473,16 +456,11 @@ class _ActorType:
|
|
|
473
456
|
An instance of the `Dataset` class for the given ID or name.
|
|
474
457
|
"""
|
|
475
458
|
self._raise_if_not_initialized()
|
|
476
|
-
self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
|
|
477
|
-
|
|
478
|
-
storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
|
|
479
|
-
|
|
480
459
|
return await Dataset.open(
|
|
481
460
|
id=id,
|
|
482
|
-
alias=alias,
|
|
483
461
|
name=name,
|
|
484
|
-
|
|
485
|
-
storage_client=
|
|
462
|
+
alias=alias,
|
|
463
|
+
storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
|
|
486
464
|
)
|
|
487
465
|
|
|
488
466
|
async def open_key_value_store(
|
|
@@ -512,16 +490,11 @@ class _ActorType:
|
|
|
512
490
|
An instance of the `KeyValueStore` class for the given ID or name.
|
|
513
491
|
"""
|
|
514
492
|
self._raise_if_not_initialized()
|
|
515
|
-
self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
|
|
516
|
-
|
|
517
|
-
storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
|
|
518
|
-
|
|
519
493
|
return await KeyValueStore.open(
|
|
520
494
|
id=id,
|
|
521
|
-
alias=alias,
|
|
522
495
|
name=name,
|
|
523
|
-
|
|
524
|
-
storage_client=
|
|
496
|
+
alias=alias,
|
|
497
|
+
storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
|
|
525
498
|
)
|
|
526
499
|
|
|
527
500
|
async def open_request_queue(
|
|
@@ -553,16 +526,11 @@ class _ActorType:
|
|
|
553
526
|
An instance of the `RequestQueue` class for the given ID or name.
|
|
554
527
|
"""
|
|
555
528
|
self._raise_if_not_initialized()
|
|
556
|
-
self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
|
|
557
|
-
|
|
558
|
-
storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
|
|
559
|
-
|
|
560
529
|
return await RequestQueue.open(
|
|
561
530
|
id=id,
|
|
562
|
-
alias=alias,
|
|
563
531
|
name=name,
|
|
564
|
-
|
|
565
|
-
storage_client=
|
|
532
|
+
alias=alias,
|
|
533
|
+
storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
|
|
566
534
|
)
|
|
567
535
|
|
|
568
536
|
@overload
|
|
@@ -142,7 +142,7 @@ class Configuration(CrawleeConfiguration):
|
|
|
142
142
|
] = None
|
|
143
143
|
|
|
144
144
|
default_dataset_id: Annotated[
|
|
145
|
-
str,
|
|
145
|
+
str | None,
|
|
146
146
|
Field(
|
|
147
147
|
validation_alias=AliasChoices(
|
|
148
148
|
'actor_default_dataset_id',
|
|
@@ -150,10 +150,10 @@ class Configuration(CrawleeConfiguration):
|
|
|
150
150
|
),
|
|
151
151
|
description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
|
|
152
152
|
),
|
|
153
|
-
] =
|
|
153
|
+
] = None
|
|
154
154
|
|
|
155
155
|
default_key_value_store_id: Annotated[
|
|
156
|
-
str,
|
|
156
|
+
str | None,
|
|
157
157
|
Field(
|
|
158
158
|
validation_alias=AliasChoices(
|
|
159
159
|
'actor_default_key_value_store_id',
|
|
@@ -161,10 +161,10 @@ class Configuration(CrawleeConfiguration):
|
|
|
161
161
|
),
|
|
162
162
|
description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
|
|
163
163
|
),
|
|
164
|
-
] =
|
|
164
|
+
] = None
|
|
165
165
|
|
|
166
166
|
default_request_queue_id: Annotated[
|
|
167
|
-
str,
|
|
167
|
+
str | None,
|
|
168
168
|
Field(
|
|
169
169
|
validation_alias=AliasChoices(
|
|
170
170
|
'actor_default_request_queue_id',
|
|
@@ -172,7 +172,7 @@ class Configuration(CrawleeConfiguration):
|
|
|
172
172
|
),
|
|
173
173
|
description='Default request queue ID for the Apify storage client when no ID or name is provided.',
|
|
174
174
|
),
|
|
175
|
-
] =
|
|
175
|
+
] = None
|
|
176
176
|
|
|
177
177
|
disable_outdated_warning: Annotated[
|
|
178
178
|
bool,
|
|
@@ -2,9 +2,11 @@ from crawlee.storage_clients import MemoryStorageClient
|
|
|
2
2
|
|
|
3
3
|
from ._apify import ApifyStorageClient
|
|
4
4
|
from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
|
|
5
|
+
from ._smart_apify import SmartApifyStorageClient
|
|
5
6
|
|
|
6
7
|
__all__ = [
|
|
7
8
|
'ApifyStorageClient',
|
|
8
9
|
'FileSystemStorageClient',
|
|
9
10
|
'MemoryStorageClient',
|
|
11
|
+
'SmartApifyStorageClient',
|
|
10
12
|
]
|
|
@@ -124,8 +124,10 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
124
124
|
)
|
|
125
125
|
apify_datasets_client = apify_client_async.datasets()
|
|
126
126
|
|
|
127
|
-
# Normalize
|
|
128
|
-
|
|
127
|
+
# Normalize unnamed default storage in cases where not defined in `configuration.default_dataset_id` to unnamed
|
|
128
|
+
# storage aliased as `__default__`
|
|
129
|
+
if not any([alias, name, id, configuration.default_dataset_id]):
|
|
130
|
+
alias = '__default__'
|
|
129
131
|
|
|
130
132
|
if alias:
|
|
131
133
|
# Check if there is pre-existing alias mapping in the default KVS.
|
|
@@ -150,6 +152,11 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
150
152
|
# If none are provided, try to get the default storage ID from environment variables.
|
|
151
153
|
elif id is None:
|
|
152
154
|
id = configuration.default_dataset_id
|
|
155
|
+
if not id:
|
|
156
|
+
raise ValueError(
|
|
157
|
+
'Dataset "id", "name", or "alias" must be specified, '
|
|
158
|
+
'or a default dataset ID must be set in the configuration.'
|
|
159
|
+
)
|
|
153
160
|
|
|
154
161
|
# Now create the client for the determined ID
|
|
155
162
|
apify_dataset_client = apify_client_async.dataset(dataset_id=id)
|
{apify-2.7.1b19 → apify-2.7.1b21}/src/apify/storage_clients/_apify/_key_value_store_client.py
RENAMED
|
@@ -115,8 +115,10 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
115
115
|
)
|
|
116
116
|
apify_kvss_client = apify_client_async.key_value_stores()
|
|
117
117
|
|
|
118
|
-
# Normalize
|
|
119
|
-
|
|
118
|
+
# Normalize unnamed default storage in cases where not defined in `configuration.default_key_value_store_id` to
|
|
119
|
+
# unnamed storage aliased as `__default__`
|
|
120
|
+
if not any([alias, name, id, configuration.default_key_value_store_id]):
|
|
121
|
+
alias = '__default__'
|
|
120
122
|
|
|
121
123
|
if alias:
|
|
122
124
|
# Check if there is pre-existing alias mapping in the default KVS.
|
|
@@ -142,6 +144,11 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
142
144
|
# If none are provided, try to get the default storage ID from environment variables.
|
|
143
145
|
elif id is None:
|
|
144
146
|
id = configuration.default_key_value_store_id
|
|
147
|
+
if not id:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
'KeyValueStore "id", "name", or "alias" must be specified, '
|
|
150
|
+
'or a default KeyValueStore ID must be set in the configuration.'
|
|
151
|
+
)
|
|
145
152
|
|
|
146
153
|
# Now create the client for the determined ID
|
|
147
154
|
apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
|