apify 2.7.1b15__tar.gz → 2.7.1b17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- {apify-2.7.1b15 → apify-2.7.1b17}/CHANGELOG.md +25 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/PKG-INFO +3 -3
- {apify-2.7.1b15 → apify-2.7.1b17}/pyproject.toml +5 -5
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_actor.py +24 -12
- apify-2.7.1b17/src/apify/events/__init__.py +5 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_dataset_client.py +38 -22
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_key_value_store_client.py +37 -22
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_request_queue_client.py +41 -27
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_storage_client.py +6 -3
- apify-2.7.1b17/src/apify/storage_clients/_apify/_utils.py +117 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_file_system/_storage_client.py +7 -1
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/requirements.txt +1 -1
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_dataset.py +83 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_key_value_store.py +83 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_request_queue.py +88 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/storage_clients/test_file_system.py +1 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/uv.lock +90 -82
- apify-2.7.1b15/src/apify/events/__init__.py +0 -5
- {apify-2.7.1b15 → apify-2.7.1b17}/.editorconfig +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/CODEOWNERS +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/build_and_deploy_docs.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/check_pr_title.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/pre_release.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/release.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/run_code_checks.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.github/workflows/update_new_issue.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.gitignore +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.markdownlint.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/.pre-commit-config.yaml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/CONTRIBUTING.md +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/LICENSE +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/Makefile +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/README.md +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/01_introduction.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/02_running_actors_locally.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/03_actor_structure.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/code/01_introduction.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/code/actor_structure/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/code/actor_structure/__main__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/code/actor_structure/main.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/01_overview/code/actor_structure/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/01_actor_lifecycle.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/02_actor_input.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/03_storages.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/04_actor_events.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/05_proxy_management.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/06_interacting_with_other_actors.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/07_webhooks.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/08_access_apify_api.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/09_running_webserver.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/10_logging.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/11_configuration.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/12_pay_per_event.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/01_context_manager.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/01_init_exit.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/01_reboot.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/01_status_message.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/02_input.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_dataset_exports.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_dataset_read_write.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_deleting_storages.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_kvs_iterating.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_kvs_public_url.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_kvs_read_write.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_opening_storages.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/03_rq.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/04_actor_events.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_apify_proxy.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_apify_proxy_config.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_custom_proxy.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_custom_proxy_function.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_proxy_actor_input.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_proxy_httpx.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/05_proxy_rotation.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/06_interacting_call.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/06_interacting_call_task.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/06_interacting_metamorph.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/06_interacting_start.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/07_webhook.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/07_webhook_preventing.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/08_actor_client.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/08_actor_new_client.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/09_webserver.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/10_log_config.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/10_logger_usage.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/10_redirect_log.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/10_redirect_log_existing_run.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/11_config.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/actor_charge.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/02_concepts/code/conditional_actor_charge.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/01_beautifulsoup_httpx.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/02_parsel_impit.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/03_playwright.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/04_selenium.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/05_crawlee.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/06_scrapy.mdx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/01_beautifulsoup_httpx.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/02_parsel_impit.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/03_playwright.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/04_selenium.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/05_crawlee_beautifulsoup.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/05_crawlee_parsel.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/05_crawlee_playwright.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/__main__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/items.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/main.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/settings.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/spiders/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/spiders/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/03_guides/code/scrapy_project/src/spiders/title.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/04_upgrading/upgrading_to_v2.md +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/04_upgrading/upgrading_to_v3.md +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/docs/pyproject.toml +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/renovate.json +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_charging.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_consts.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_crypto.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_models.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_proxy_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/_utils.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/events/_apify_event_manager.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/events/_types.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/events/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/log.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/request_loaders/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/request_loaders/_apify_request_list.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/request_loaders/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/_actor_runner.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/_async_thread.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/_logging_config.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/extensions/_httpcache.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/middlewares/apify_proxy.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/middlewares/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/pipelines/actor_dataset_push.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/pipelines/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/requests.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/scheduler.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/scrapy/utils.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_models.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_file_system/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_file_system/_key_value_store_client.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storages/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storages/py.typed +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/README.md +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/_utils.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/Dockerfile +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/server.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/src/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/src/__main__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/actor_source_base/src/main.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/conftest.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_api_helpers.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_call_timeouts.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_charge.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_events.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_log.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_actor_scrapy.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_crawlers_with_storages.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_fixtures.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/integration/test_request_queue.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_dataset.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_env_helpers.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_helpers.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_key_value_store.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_lifecycle.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_log.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_non_default_instance.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_actor_request_queue.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/actor/test_request_list.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/conftest.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/events/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/events/test_apify_event_manager.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/extensions/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/extensions/test_httpcache.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/middlewares/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/middlewares/test_apify_proxy.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/pipelines/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/pipelines/test_actor_dataset_push.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/requests/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/requests/test_to_apify_request.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/requests/test_to_scrapy_request.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/utils/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/utils/test_apply_apify_settings.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/scrapy/utils/test_get_basic_auth_header.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/storage_clients/__init__.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/storage_clients/test_apify_request_queue_client.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/test_crypto.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/tests/unit/test_proxy_configuration.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/.eslintrc.json +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/babel.config.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/build_api_reference.sh +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/docusaurus.config.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/generate_module_shortcuts.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/package-lock.json +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/package.json +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/sidebars.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/ApiLink.jsx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/Gradients.jsx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/Highlights.jsx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/Highlights.module.css +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/RunnableCodeBlock.jsx +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/components/RunnableCodeBlock.module.css +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/css/custom.css +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/pages/home_page_example.py +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/pages/index.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/pages/index.module.css +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/src/theme/DocItem/Content/index.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/static/.nojekyll +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/static/img/docs-og.png +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/static/img/guides/redirected_logs_example.webp +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/tools/docs-prettier.config.js +0 -0
- {apify-2.7.1b15 → apify-2.7.1b17}/website/tools/utils/externalLink.js +0 -0
|
@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
|
|
|
9
9
|
|
|
10
10
|
- Add deduplication to `add_batch_of_requests` ([#534](https://github.com/apify/apify-sdk-python/pull/534)) ([dd03c4d](https://github.com/apify/apify-sdk-python/commit/dd03c4d446f611492adf35f1b5738648ee5a66f7)) by [@Pijukatel](https://github.com/Pijukatel), closes [#514](https://github.com/apify/apify-sdk-python/issues/514)
|
|
11
11
|
- Add new methods to ChargingManager ([#580](https://github.com/apify/apify-sdk-python/pull/580)) ([54f7f8b](https://github.com/apify/apify-sdk-python/commit/54f7f8b29c5982be98b595dac11eceff915035c9)) by [@vdusek](https://github.com/vdusek)
|
|
12
|
+
- Add support for NDU storages ([#594](https://github.com/apify/apify-sdk-python/pull/594)) ([8721ef5](https://github.com/apify/apify-sdk-python/commit/8721ef5731bcb1a04ad63c930089bf83be29f308)) by [@vdusek](https://github.com/vdusek), closes [#1175](https://github.com/apify/apify-sdk-python/issues/1175)
|
|
12
13
|
|
|
13
14
|
### 🐛 Bug Fixes
|
|
14
15
|
|
|
@@ -114,6 +115,30 @@ All notable changes to this project will be documented in this file.
|
|
|
114
115
|
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
115
116
|
|
|
116
117
|
|
|
118
|
+
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
119
|
+
|
|
120
|
+
### 🚀 Features
|
|
121
|
+
|
|
122
|
+
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
123
|
+
|
|
124
|
+
### 🐛 Bug Fixes
|
|
125
|
+
|
|
126
|
+
- Sync `@docusaurus` theme version [internal] ([#500](https://github.com/apify/apify-sdk-python/pull/500)) ([a7485e7](https://github.com/apify/apify-sdk-python/commit/a7485e7d2276fde464ce862573d5b95e7d4d836a)) by [@katzino](https://github.com/katzino)
|
|
127
|
+
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
|
|
131
|
+
|
|
132
|
+
### 🚀 Features
|
|
133
|
+
|
|
134
|
+
- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
|
|
135
|
+
|
|
136
|
+
### 🐛 Bug Fixes
|
|
137
|
+
|
|
138
|
+
- Sync `@docusaurus` theme version [internal] ([#500](https://github.com/apify/apify-sdk-python/pull/500)) ([a7485e7](https://github.com/apify/apify-sdk-python/commit/a7485e7d2276fde464ce862573d5b95e7d4d836a)) by [@katzino](https://github.com/katzino)
|
|
139
|
+
- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
|
|
140
|
+
|
|
141
|
+
|
|
117
142
|
|
|
118
143
|
## [2.7.3](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.3) (2025-08-11)
|
|
119
144
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apify
|
|
3
|
-
Version: 2.7.
|
|
3
|
+
Version: 2.7.1b17
|
|
4
4
|
Summary: Apify SDK for Python
|
|
5
5
|
Project-URL: Apify Homepage, https://apify.com
|
|
6
6
|
Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
|
|
@@ -228,9 +228,9 @@ Requires-Python: >=3.10
|
|
|
228
228
|
Requires-Dist: apify-client<3.0.0,>=2.0.0
|
|
229
229
|
Requires-Dist: apify-shared<3.0.0,>=2.0.0
|
|
230
230
|
Requires-Dist: cachetools>=5.5.0
|
|
231
|
-
Requires-Dist: crawlee==
|
|
231
|
+
Requires-Dist: crawlee==0.6.13b37
|
|
232
232
|
Requires-Dist: cryptography>=42.0.0
|
|
233
|
-
Requires-Dist: impit>=0.
|
|
233
|
+
Requires-Dist: impit>=0.6.1
|
|
234
234
|
Requires-Dist: lazy-object-proxy>=1.11.0
|
|
235
235
|
Requires-Dist: more-itertools>=10.2.0
|
|
236
236
|
Requires-Dist: typing-extensions>=4.1.0
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "apify"
|
|
7
|
-
version = "2.7.
|
|
7
|
+
version = "2.7.1b17"
|
|
8
8
|
description = "Apify SDK for Python"
|
|
9
9
|
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -36,10 +36,10 @@ keywords = [
|
|
|
36
36
|
dependencies = [
|
|
37
37
|
"apify-client>=2.0.0,<3.0.0",
|
|
38
38
|
"apify-shared>=2.0.0,<3.0.0",
|
|
39
|
-
"crawlee==
|
|
39
|
+
"crawlee==0.6.13b37",
|
|
40
40
|
"cachetools>=5.5.0",
|
|
41
41
|
"cryptography>=42.0.0",
|
|
42
|
-
"impit>=0.
|
|
42
|
+
"impit>=0.6.1",
|
|
43
43
|
"lazy-object-proxy>=1.11.0",
|
|
44
44
|
"more_itertools>=10.2.0",
|
|
45
45
|
"typing-extensions>=4.1.0",
|
|
@@ -66,10 +66,10 @@ dev = [
|
|
|
66
66
|
"crawlee[parsel]",
|
|
67
67
|
"dycw-pytest-only~=2.1.0",
|
|
68
68
|
"griffe",
|
|
69
|
-
"mypy~=1.
|
|
69
|
+
"mypy~=1.18.1",
|
|
70
70
|
"pre-commit~=4.3.0",
|
|
71
71
|
"pydoc-markdown~=4.8.0",
|
|
72
|
-
"pytest-asyncio~=1.
|
|
72
|
+
"pytest-asyncio~=1.2.0",
|
|
73
73
|
"pytest-cov~=7.0.0",
|
|
74
74
|
"pytest-httpserver~=1.1.0",
|
|
75
75
|
"pytest-timeout~=2.4.0",
|
|
@@ -401,6 +401,7 @@ class _ActorType:
|
|
|
401
401
|
self,
|
|
402
402
|
*,
|
|
403
403
|
id: str | None = None,
|
|
404
|
+
alias: str | None = None,
|
|
404
405
|
name: str | None = None,
|
|
405
406
|
force_cloud: bool = False,
|
|
406
407
|
) -> Dataset:
|
|
@@ -411,10 +412,12 @@ class _ActorType:
|
|
|
411
412
|
the Apify cloud.
|
|
412
413
|
|
|
413
414
|
Args:
|
|
414
|
-
id: ID of the dataset to
|
|
415
|
-
|
|
416
|
-
name:
|
|
417
|
-
|
|
415
|
+
id: The ID of the dataset to open. If provided, searches for existing dataset by ID.
|
|
416
|
+
Mutually exclusive with name and alias.
|
|
417
|
+
name: The name of the dataset to open (global scope, persists across runs).
|
|
418
|
+
Mutually exclusive with id and alias.
|
|
419
|
+
alias: The alias of the dataset to open (run scope, creates unnamed storage).
|
|
420
|
+
Mutually exclusive with id and name.
|
|
418
421
|
force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
|
|
419
422
|
to combine local and cloud storage.
|
|
420
423
|
|
|
@@ -428,6 +431,7 @@ class _ActorType:
|
|
|
428
431
|
|
|
429
432
|
return await Dataset.open(
|
|
430
433
|
id=id,
|
|
434
|
+
alias=alias,
|
|
431
435
|
name=name,
|
|
432
436
|
configuration=self._configuration,
|
|
433
437
|
storage_client=storage_client,
|
|
@@ -437,6 +441,7 @@ class _ActorType:
|
|
|
437
441
|
self,
|
|
438
442
|
*,
|
|
439
443
|
id: str | None = None,
|
|
444
|
+
alias: str | None = None,
|
|
440
445
|
name: str | None = None,
|
|
441
446
|
force_cloud: bool = False,
|
|
442
447
|
) -> KeyValueStore:
|
|
@@ -446,10 +451,12 @@ class _ActorType:
|
|
|
446
451
|
and retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.
|
|
447
452
|
|
|
448
453
|
Args:
|
|
449
|
-
id: ID of the
|
|
450
|
-
|
|
451
|
-
name:
|
|
452
|
-
|
|
454
|
+
id: The ID of the KVS to open. If provided, searches for existing KVS by ID.
|
|
455
|
+
Mutually exclusive with name and alias.
|
|
456
|
+
name: The name of the KVS to open (global scope, persists across runs).
|
|
457
|
+
Mutually exclusive with id and alias.
|
|
458
|
+
alias: The alias of the KVS to open (run scope, creates unnamed storage).
|
|
459
|
+
Mutually exclusive with id and name.
|
|
453
460
|
force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
|
|
454
461
|
to combine local and cloud storage.
|
|
455
462
|
|
|
@@ -462,6 +469,7 @@ class _ActorType:
|
|
|
462
469
|
|
|
463
470
|
return await KeyValueStore.open(
|
|
464
471
|
id=id,
|
|
472
|
+
alias=alias,
|
|
465
473
|
name=name,
|
|
466
474
|
configuration=self._configuration,
|
|
467
475
|
storage_client=storage_client,
|
|
@@ -471,6 +479,7 @@ class _ActorType:
|
|
|
471
479
|
self,
|
|
472
480
|
*,
|
|
473
481
|
id: str | None = None,
|
|
482
|
+
alias: str | None = None,
|
|
474
483
|
name: str | None = None,
|
|
475
484
|
force_cloud: bool = False,
|
|
476
485
|
) -> RequestQueue:
|
|
@@ -482,10 +491,12 @@ class _ActorType:
|
|
|
482
491
|
crawling orders.
|
|
483
492
|
|
|
484
493
|
Args:
|
|
485
|
-
id: ID of the
|
|
486
|
-
|
|
487
|
-
name:
|
|
488
|
-
|
|
494
|
+
id: The ID of the RQ to open. If provided, searches for existing RQ by ID.
|
|
495
|
+
Mutually exclusive with name and alias.
|
|
496
|
+
name: The name of the RQ to open (global scope, persists across runs).
|
|
497
|
+
Mutually exclusive with id and alias.
|
|
498
|
+
alias: The alias of the RQ to open (run scope, creates unnamed storage).
|
|
499
|
+
Mutually exclusive with id and name.
|
|
489
500
|
force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
|
|
490
501
|
to combine local and cloud storage.
|
|
491
502
|
|
|
@@ -499,6 +510,7 @@ class _ActorType:
|
|
|
499
510
|
|
|
500
511
|
return await RequestQueue.open(
|
|
501
512
|
id=id,
|
|
513
|
+
alias=alias,
|
|
502
514
|
name=name,
|
|
503
515
|
configuration=self._configuration,
|
|
504
516
|
storage_client=storage_client,
|
|
@@ -12,6 +12,8 @@ from crawlee._utils.file import json_dumps
|
|
|
12
12
|
from crawlee.storage_clients._base import DatasetClient
|
|
13
13
|
from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
|
|
14
14
|
|
|
15
|
+
from ._utils import resolve_alias_to_id, store_alias_mapping
|
|
16
|
+
|
|
15
17
|
if TYPE_CHECKING:
|
|
16
18
|
from collections.abc import AsyncIterator
|
|
17
19
|
|
|
@@ -66,6 +68,7 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
66
68
|
*,
|
|
67
69
|
id: str | None,
|
|
68
70
|
name: str | None,
|
|
71
|
+
alias: str | None,
|
|
69
72
|
configuration: Configuration,
|
|
70
73
|
) -> ApifyDatasetClient:
|
|
71
74
|
"""Open an Apify dataset client.
|
|
@@ -74,22 +77,27 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
74
77
|
It handles authentication, storage lookup/creation, and metadata retrieval.
|
|
75
78
|
|
|
76
79
|
Args:
|
|
77
|
-
id: The ID of
|
|
78
|
-
|
|
79
|
-
name: The name of
|
|
80
|
-
|
|
80
|
+
id: The ID of the dataset to open. If provided, searches for existing dataset by ID.
|
|
81
|
+
Mutually exclusive with name and alias.
|
|
82
|
+
name: The name of the dataset to open (global scope, persists across runs).
|
|
83
|
+
Mutually exclusive with id and alias.
|
|
84
|
+
alias: The alias of the dataset to open (run scope, creates unnamed storage).
|
|
85
|
+
Mutually exclusive with id and name.
|
|
81
86
|
configuration: The configuration object containing API credentials and settings. Must include a valid
|
|
82
87
|
`token` and `api_base_url`. May also contain a `default_dataset_id` for fallback when neither
|
|
83
|
-
`id` nor `
|
|
88
|
+
`id`, `name`, nor `alias` is provided.
|
|
84
89
|
|
|
85
90
|
Returns:
|
|
86
91
|
An instance for the opened or created storage client.
|
|
87
92
|
|
|
88
93
|
Raises:
|
|
89
|
-
ValueError: If the configuration is missing required fields (token, api_base_url), if
|
|
90
|
-
|
|
91
|
-
the configuration.
|
|
94
|
+
ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
|
|
95
|
+
`id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
|
|
96
|
+
in the configuration.
|
|
92
97
|
"""
|
|
98
|
+
if sum(1 for param in [id, name, alias] if param is not None) > 1:
|
|
99
|
+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
|
|
100
|
+
|
|
93
101
|
token = configuration.token
|
|
94
102
|
if not token:
|
|
95
103
|
raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
|
|
@@ -115,27 +123,35 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
115
123
|
)
|
|
116
124
|
apify_datasets_client = apify_client_async.datasets()
|
|
117
125
|
|
|
118
|
-
#
|
|
119
|
-
if
|
|
120
|
-
raise ValueError('Only one of "id" or "name" can be specified, not both.')
|
|
126
|
+
# Normalize 'default' alias to None
|
|
127
|
+
alias = None if alias == 'default' else alias
|
|
121
128
|
|
|
122
|
-
#
|
|
123
|
-
if
|
|
124
|
-
|
|
129
|
+
# Handle alias resolution
|
|
130
|
+
if alias:
|
|
131
|
+
# Try to resolve alias to existing storage ID
|
|
132
|
+
resolved_id = await resolve_alias_to_id(alias, 'dataset', configuration)
|
|
133
|
+
if resolved_id:
|
|
134
|
+
id = resolved_id
|
|
135
|
+
else:
|
|
136
|
+
# Create a new storage and store the alias mapping
|
|
137
|
+
new_storage_metadata = DatasetMetadata.model_validate(
|
|
138
|
+
await apify_datasets_client.get_or_create(),
|
|
139
|
+
)
|
|
140
|
+
id = new_storage_metadata.id
|
|
141
|
+
await store_alias_mapping(alias, 'dataset', id, configuration)
|
|
125
142
|
|
|
126
143
|
# If name is provided, get or create the storage by name.
|
|
127
|
-
|
|
144
|
+
elif name:
|
|
128
145
|
id = DatasetMetadata.model_validate(
|
|
129
146
|
await apify_datasets_client.get_or_create(name=name),
|
|
130
147
|
).id
|
|
131
|
-
apify_dataset_client = apify_client_async.dataset(dataset_id=id)
|
|
132
148
|
|
|
133
|
-
# If
|
|
134
|
-
|
|
135
|
-
# a new storage ID after Actor's reboot or migration.
|
|
136
|
-
if id is None and name is None:
|
|
149
|
+
# If none are provided, try to get the default storage ID from environment variables.
|
|
150
|
+
elif id is None:
|
|
137
151
|
id = configuration.default_dataset_id
|
|
138
|
-
|
|
152
|
+
|
|
153
|
+
# Now create the client for the determined ID
|
|
154
|
+
apify_dataset_client = apify_client_async.dataset(dataset_id=id)
|
|
139
155
|
|
|
140
156
|
# Fetch its metadata.
|
|
141
157
|
metadata = await apify_dataset_client.get()
|
|
@@ -150,7 +166,7 @@ class ApifyDatasetClient(DatasetClient):
|
|
|
150
166
|
# Verify that the storage exists by fetching its metadata again.
|
|
151
167
|
metadata = await apify_dataset_client.get()
|
|
152
168
|
if metadata is None:
|
|
153
|
-
raise ValueError(f'Opening dataset with id={id}
|
|
169
|
+
raise ValueError(f'Opening dataset with id={id}, name={name}, and alias={alias} failed.')
|
|
154
170
|
|
|
155
171
|
return cls(
|
|
156
172
|
api_client=apify_dataset_client,
|
{apify-2.7.1b15 → apify-2.7.1b17}/src/apify/storage_clients/_apify/_key_value_store_client.py
RENAMED
|
@@ -12,6 +12,7 @@ from crawlee.storage_clients._base import KeyValueStoreClient
|
|
|
12
12
|
from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
|
|
13
13
|
|
|
14
14
|
from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
|
|
15
|
+
from ._utils import resolve_alias_to_id, store_alias_mapping
|
|
15
16
|
from apify._crypto import create_hmac_signature
|
|
16
17
|
|
|
17
18
|
if TYPE_CHECKING:
|
|
@@ -58,6 +59,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
58
59
|
*,
|
|
59
60
|
id: str | None,
|
|
60
61
|
name: str | None,
|
|
62
|
+
alias: str | None,
|
|
61
63
|
configuration: Configuration,
|
|
62
64
|
) -> ApifyKeyValueStoreClient:
|
|
63
65
|
"""Open an Apify key-value store client.
|
|
@@ -66,22 +68,27 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
66
68
|
It handles authentication, storage lookup/creation, and metadata retrieval.
|
|
67
69
|
|
|
68
70
|
Args:
|
|
69
|
-
id: The ID of
|
|
70
|
-
|
|
71
|
-
name: The name of
|
|
72
|
-
|
|
71
|
+
id: The ID of the KVS to open. If provided, searches for existing KVS by ID.
|
|
72
|
+
Mutually exclusive with name and alias.
|
|
73
|
+
name: The name of the KVS to open (global scope, persists across runs).
|
|
74
|
+
Mutually exclusive with id and alias.
|
|
75
|
+
alias: The alias of the KVS to open (run scope, creates unnamed storage).
|
|
76
|
+
Mutually exclusive with id and name.
|
|
73
77
|
configuration: The configuration object containing API credentials and settings. Must include a valid
|
|
74
78
|
`token` and `api_base_url`. May also contain a `default_key_value_store_id` for fallback when
|
|
75
|
-
neither `id` nor `
|
|
79
|
+
neither `id`, `name`, nor `alias` is provided.
|
|
76
80
|
|
|
77
81
|
Returns:
|
|
78
82
|
An instance for the opened or created storage client.
|
|
79
83
|
|
|
80
84
|
Raises:
|
|
81
|
-
ValueError: If the configuration is missing required fields (token, api_base_url), if
|
|
82
|
-
|
|
85
|
+
ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
|
|
86
|
+
`id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
|
|
83
87
|
in the configuration.
|
|
84
88
|
"""
|
|
89
|
+
if sum(1 for param in [id, name, alias] if param is not None) > 1:
|
|
90
|
+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
|
|
91
|
+
|
|
85
92
|
token = configuration.token
|
|
86
93
|
if not token:
|
|
87
94
|
raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
|
|
@@ -107,27 +114,35 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
107
114
|
)
|
|
108
115
|
apify_kvss_client = apify_client_async.key_value_stores()
|
|
109
116
|
|
|
110
|
-
#
|
|
111
|
-
if
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
+
# Normalize 'default' alias to None
|
|
118
|
+
alias = None if alias == 'default' else alias
|
|
119
|
+
|
|
120
|
+
# Handle alias resolution
|
|
121
|
+
if alias:
|
|
122
|
+
# Try to resolve alias to existing storage ID
|
|
123
|
+
resolved_id = await resolve_alias_to_id(alias, 'kvs', configuration)
|
|
124
|
+
if resolved_id:
|
|
125
|
+
id = resolved_id
|
|
126
|
+
else:
|
|
127
|
+
# Create a new storage and store the alias mapping
|
|
128
|
+
new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
|
|
129
|
+
await apify_kvss_client.get_or_create(),
|
|
130
|
+
)
|
|
131
|
+
id = new_storage_metadata.id
|
|
132
|
+
await store_alias_mapping(alias, 'kvs', id, configuration)
|
|
117
133
|
|
|
118
134
|
# If name is provided, get or create the storage by name.
|
|
119
|
-
|
|
135
|
+
elif name:
|
|
120
136
|
id = ApifyKeyValueStoreMetadata.model_validate(
|
|
121
137
|
await apify_kvss_client.get_or_create(name=name),
|
|
122
138
|
).id
|
|
123
|
-
apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
|
|
124
139
|
|
|
125
|
-
# If
|
|
126
|
-
|
|
127
|
-
# a new storage ID after Actor's reboot or migration.
|
|
128
|
-
if id is None and name is None:
|
|
140
|
+
# If none are provided, try to get the default storage ID from environment variables.
|
|
141
|
+
elif id is None:
|
|
129
142
|
id = configuration.default_key_value_store_id
|
|
130
|
-
|
|
143
|
+
|
|
144
|
+
# Now create the client for the determined ID
|
|
145
|
+
apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
|
|
131
146
|
|
|
132
147
|
# Fetch its metadata.
|
|
133
148
|
metadata = await apify_kvs_client.get()
|
|
@@ -142,7 +157,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
|
|
|
142
157
|
# Verify that the storage exists by fetching its metadata again.
|
|
143
158
|
metadata = await apify_kvs_client.get()
|
|
144
159
|
if metadata is None:
|
|
145
|
-
raise ValueError(f'Opening key-value store with id={id}
|
|
160
|
+
raise ValueError(f'Opening key-value store with id={id}, name={name}, and alias={alias} failed.')
|
|
146
161
|
|
|
147
162
|
return cls(
|
|
148
163
|
api_client=apify_kvs_client,
|
|
@@ -18,6 +18,7 @@ from crawlee.storage_clients._base import RequestQueueClient
|
|
|
18
18
|
from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
|
|
19
19
|
|
|
20
20
|
from ._models import CachedRequest, ProlongRequestLockResponse, RequestQueueHead
|
|
21
|
+
from ._utils import resolve_alias_to_id, store_alias_mapping
|
|
21
22
|
from apify import Request
|
|
22
23
|
|
|
23
24
|
if TYPE_CHECKING:
|
|
@@ -135,6 +136,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
|
|
|
135
136
|
*,
|
|
136
137
|
id: str | None,
|
|
137
138
|
name: str | None,
|
|
139
|
+
alias: str | None,
|
|
138
140
|
configuration: Configuration,
|
|
139
141
|
) -> ApifyRequestQueueClient:
|
|
140
142
|
"""Open an Apify request queue client.
|
|
@@ -144,22 +146,27 @@ class ApifyRequestQueueClient(RequestQueueClient):
|
|
|
144
146
|
management structures.
|
|
145
147
|
|
|
146
148
|
Args:
|
|
147
|
-
id: The ID of
|
|
148
|
-
|
|
149
|
-
name: The name of
|
|
150
|
-
|
|
149
|
+
id: The ID of the RQ to open. If provided, searches for existing RQ by ID.
|
|
150
|
+
Mutually exclusive with name and alias.
|
|
151
|
+
name: The name of the RQ to open (global scope, persists across runs).
|
|
152
|
+
Mutually exclusive with id and alias.
|
|
153
|
+
alias: The alias of the RQ to open (run scope, creates unnamed storage).
|
|
154
|
+
Mutually exclusive with id and name.
|
|
151
155
|
configuration: The configuration object containing API credentials and settings. Must include a valid
|
|
152
156
|
`token` and `api_base_url`. May also contain a `default_request_queue_id` for fallback when neither
|
|
153
|
-
`id` nor `
|
|
157
|
+
`id`, `name`, nor `alias` is provided.
|
|
154
158
|
|
|
155
159
|
Returns:
|
|
156
160
|
An instance for the opened or created storage client.
|
|
157
161
|
|
|
158
162
|
Raises:
|
|
159
|
-
ValueError: If the configuration is missing required fields (token, api_base_url), if
|
|
160
|
-
|
|
163
|
+
ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
|
|
164
|
+
`id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
|
|
161
165
|
in the configuration.
|
|
162
166
|
"""
|
|
167
|
+
if sum(1 for param in [id, name, alias] if param is not None) > 1:
|
|
168
|
+
raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
|
|
169
|
+
|
|
163
170
|
token = configuration.token
|
|
164
171
|
if not token:
|
|
165
172
|
raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
|
|
@@ -185,25 +192,32 @@ class ApifyRequestQueueClient(RequestQueueClient):
|
|
|
185
192
|
)
|
|
186
193
|
apify_rqs_client = apify_client_async.request_queues()
|
|
187
194
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
id =
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
195
|
+
# Normalize 'default' alias to None
|
|
196
|
+
alias = None if alias == 'default' else alias
|
|
197
|
+
|
|
198
|
+
# Handle alias resolution
|
|
199
|
+
if alias:
|
|
200
|
+
# Try to resolve alias to existing storage ID
|
|
201
|
+
resolved_id = await resolve_alias_to_id(alias, 'rq', configuration)
|
|
202
|
+
if resolved_id:
|
|
203
|
+
id = resolved_id
|
|
204
|
+
else:
|
|
205
|
+
# Create a new storage and store the alias mapping
|
|
206
|
+
new_storage_metadata = RequestQueueMetadata.model_validate(
|
|
207
|
+
await apify_rqs_client.get_or_create(),
|
|
208
|
+
)
|
|
209
|
+
id = new_storage_metadata.id
|
|
210
|
+
await store_alias_mapping(alias, 'rq', id, configuration)
|
|
211
|
+
|
|
212
|
+
# If name is provided, get or create the storage by name.
|
|
213
|
+
elif name:
|
|
214
|
+
id = RequestQueueMetadata.model_validate(
|
|
215
|
+
await apify_rqs_client.get_or_create(name=name),
|
|
216
|
+
).id
|
|
217
|
+
|
|
218
|
+
# If none are provided, try to get the default storage ID from environment variables.
|
|
219
|
+
elif id is None:
|
|
220
|
+
id = configuration.default_request_queue_id
|
|
207
221
|
|
|
208
222
|
# Use suitable client_key to make `hadMultipleClients` response of Apify API useful.
|
|
209
223
|
# It should persist across migrated or resurrected Actor runs on the Apify platform.
|
|
@@ -227,7 +241,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
|
|
|
227
241
|
# Verify that the storage exists by fetching its metadata again.
|
|
228
242
|
metadata = await apify_rq_client.get()
|
|
229
243
|
if metadata is None:
|
|
230
|
-
raise ValueError(f'Opening request queue with id={id}
|
|
244
|
+
raise ValueError(f'Opening request queue with id={id}, name={name}, and alias={alias} failed.')
|
|
231
245
|
|
|
232
246
|
metadata_model = RequestQueueMetadata.model_validate(metadata)
|
|
233
247
|
|
|
@@ -25,6 +25,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
25
25
|
*,
|
|
26
26
|
id: str | None = None,
|
|
27
27
|
name: str | None = None,
|
|
28
|
+
alias: str | None = None,
|
|
28
29
|
configuration: Configuration | None = None,
|
|
29
30
|
) -> ApifyDatasetClient:
|
|
30
31
|
# Import here to avoid circular imports.
|
|
@@ -32,7 +33,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
32
33
|
|
|
33
34
|
configuration = configuration or ApifyConfiguration.get_global_configuration()
|
|
34
35
|
if isinstance(configuration, ApifyConfiguration):
|
|
35
|
-
return await ApifyDatasetClient.open(id=id, name=name, configuration=configuration)
|
|
36
|
+
return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
|
|
36
37
|
|
|
37
38
|
raise TypeError(
|
|
38
39
|
f'Expected "configuration" to be an instance of "apify.Configuration", '
|
|
@@ -45,6 +46,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
45
46
|
*,
|
|
46
47
|
id: str | None = None,
|
|
47
48
|
name: str | None = None,
|
|
49
|
+
alias: str | None = None,
|
|
48
50
|
configuration: Configuration | None = None,
|
|
49
51
|
) -> ApifyKeyValueStoreClient:
|
|
50
52
|
# Import here to avoid circular imports.
|
|
@@ -52,7 +54,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
52
54
|
|
|
53
55
|
configuration = configuration or ApifyConfiguration.get_global_configuration()
|
|
54
56
|
if isinstance(configuration, ApifyConfiguration):
|
|
55
|
-
return await ApifyKeyValueStoreClient.open(id=id, name=name, configuration=configuration)
|
|
57
|
+
return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
|
|
56
58
|
|
|
57
59
|
raise TypeError(
|
|
58
60
|
f'Expected "configuration" to be an instance of "apify.Configuration", '
|
|
@@ -65,6 +67,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
65
67
|
*,
|
|
66
68
|
id: str | None = None,
|
|
67
69
|
name: str | None = None,
|
|
70
|
+
alias: str | None = None,
|
|
68
71
|
configuration: Configuration | None = None,
|
|
69
72
|
) -> ApifyRequestQueueClient:
|
|
70
73
|
# Import here to avoid circular imports.
|
|
@@ -72,7 +75,7 @@ class ApifyStorageClient(StorageClient):
|
|
|
72
75
|
|
|
73
76
|
configuration = configuration or ApifyConfiguration.get_global_configuration()
|
|
74
77
|
if isinstance(configuration, ApifyConfiguration):
|
|
75
|
-
return await ApifyRequestQueueClient.open(id=id, name=name, configuration=configuration)
|
|
78
|
+
return await ApifyRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
|
|
76
79
|
|
|
77
80
|
raise TypeError(
|
|
78
81
|
f'Expected "configuration" to be an instance of "apify.Configuration", '
|