apify 2.3.1b2__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- {apify-2.3.1b2 → apify-2.4.0}/CHANGELOG.md +10 -0
- {apify-2.3.1b2 → apify-2.4.0}/PKG-INFO +4 -4
- apify-2.4.0/docs/03_concepts/12_pay_per_event.mdx +46 -0
- apify-2.4.0/docs/03_concepts/code/actor_charge.py +30 -0
- apify-2.4.0/docs/03_concepts/code/conditional_actor_charge.py +18 -0
- {apify-2.3.1b2 → apify-2.4.0}/pyproject.toml +5 -7
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_actor.py +40 -11
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_crypto.py +38 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_platform_event_manager.py +3 -3
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_apify_storage_client.py +3 -3
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_dataset_client.py +1 -1
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_dataset_collection_client.py +1 -1
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_key_value_store_client.py +18 -3
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_key_value_store_collection_client.py +1 -1
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_request_queue_client.py +1 -1
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_request_queue_collection_client.py +1 -1
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/storages/_request_list.py +4 -4
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_key_value_store.py +15 -6
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_log.py +5 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_lifecycle.py +4 -4
- apify-2.4.0/tests/unit/actor/test_actor_log.py +104 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/test_crypto.py +31 -1
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/test_platform_event_manager.py +7 -7
- {apify-2.3.1b2 → apify-2.4.0}/uv.lock +174 -378
- {apify-2.3.1b2 → apify-2.4.0}/website/package-lock.json +274 -249
- apify-2.4.0/website/src/components/ApiLink.jsx +10 -0
- apify-2.4.0/website/src/components/Gradients.jsx +20 -0
- apify-2.4.0/website/src/components/Highlights.jsx +104 -0
- apify-2.4.0/website/src/components/Highlights.module.css +46 -0
- apify-2.4.0/website/src/components/RunnableCodeBlock.jsx +44 -0
- apify-2.4.0/website/src/components/RunnableCodeBlock.module.css +39 -0
- apify-2.3.1b2/tests/unit/actor/test_actor_log.py +0 -93
- {apify-2.3.1b2 → apify-2.4.0}/.editorconfig +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/CODEOWNERS +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/build_and_deploy_docs.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/check_pr_title.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/pre_release.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/release.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/run_code_checks.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.github/workflows/update_new_issue.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.gitignore +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.markdownlint.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/.pre-commit-config.yaml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/CONTRIBUTING.md +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/LICENSE +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/Makefile +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/README.md +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/01_introduction.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/02_running_actors_locally.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/03_actor_structure.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/code/01_introduction.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/code/actor_structure/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/code/actor_structure/__main__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/code/actor_structure/main.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/01_overview/code/actor_structure/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/01_beautifulsoup_httpx.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/02_crawlee.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/03_playwright.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/04_selenium.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/05_scrapy.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/01_beautifulsoup_httpx.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/02_crawlee_beautifulsoup.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/02_crawlee_playwright.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/03_playwright.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/04_selenium.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/__main__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/items.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/main.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/settings.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/spiders/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/spiders/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/02_guides/code/scrapy_project/src/spiders/title.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/01_actor_lifecycle.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/02_actor_input.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/03_storages.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/04_actor_events.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/05_proxy_management.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/06_interacting_with_other_actors.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/07_webhooks.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/08_access_apify_api.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/09_running_webserver.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/10_logging.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/11_configuration.mdx +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/01_context_manager.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/01_init_exit.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/01_reboot.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/01_status_message.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/02_input.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_dataset_exports.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_dataset_read_write.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_deleting_storages.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_kvs_iterating.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_kvs_public_url.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_kvs_read_write.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_opening_storages.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/03_rq.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/04_actor_events.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_apify_proxy.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_apify_proxy_config.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_custom_proxy.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_custom_proxy_function.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_proxy_actor_input.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_proxy_httpx.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/05_proxy_rotation.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/06_interacting_call.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/06_interacting_call_task.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/06_interacting_metamorph.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/06_interacting_start.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/07_webhook.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/07_webhook_preventing.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/08_actor_client.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/08_actor_new_client.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/09_webserver.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/10_log_config.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/10_logger_usage.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/03_concepts/code/11_config.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/04_upgrading/upgrading_to_v2.md +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/docs/pyproject.toml +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/renovate.json +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_charging.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_configuration.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_consts.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_models.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_proxy_configuration.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/_utils.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/log.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/_actor_runner.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/_async_thread.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/_logging_config.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/middlewares/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/middlewares/apify_proxy.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/middlewares/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/pipelines/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/pipelines/actor_dataset_push.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/pipelines/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/requests.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/scheduler.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/scrapy/utils.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/storages/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/src/apify/storages/py.typed +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/README.md +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/_utils.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/actor_source_base/Dockerfile +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/actor_source_base/requirements.txt +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/actor_source_base/src/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/actor_source_base/src/__main__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/actor_source_base/src/main.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/conftest.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_api_helpers.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_charge.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_dataset.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_events.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_lifecycle.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_request_queue.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_actor_scrapy.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_fixtures.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/integration/test_request_queue.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_create_proxy_configuration.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_dataset.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_env_helpers.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_helpers.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_key_value_store.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_non_default_instance.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_actor_request_queue.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/actor/test_request_list.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/conftest.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/middlewares/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/middlewares/test_apify_proxy.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/pipelines/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/pipelines/test_actor_dataset_push.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/requests/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/requests/test_to_apify_request.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/requests/test_to_scrapy_request.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/utils/__init__.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/utils/test_apply_apify_settings.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/scrapy/utils/test_get_basic_auth_header.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/tests/unit/test_proxy_configuration.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/.eslintrc.json +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/babel.config.js +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/build_api_reference.sh +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/docusaurus.config.js +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/generate_module_shortcuts.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/package.json +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/sidebars.js +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/src/css/custom.css +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/src/pages/home_page_example.py +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/src/pages/index.js +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/src/pages/index.module.css +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/static/.nojekyll +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/static/img/docs-og.png +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/tools/docs-prettier.config.js +0 -0
- {apify-2.3.1b2 → apify-2.4.0}/website/tools/utils/externalLink.js +0 -0
|
@@ -2,6 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [2.4.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.4.0) (2025-03-07)
|
|
6
|
+
|
|
7
|
+
### 🚀 Features
|
|
8
|
+
|
|
9
|
+
- Update to Crawlee v0.6 ([#420](https://github.com/apify/apify-sdk-python/pull/420)) ([9be4336](https://github.com/apify/apify-sdk-python/commit/9be433667231cc5739861fa693d7a726860d6aca)) by [@vdusek](https://github.com/vdusek)
|
|
10
|
+
- Add Actor `exit_process` option ([#424](https://github.com/apify/apify-sdk-python/pull/424)) ([994c832](https://github.com/apify/apify-sdk-python/commit/994c8323b994e009db0ccdcb624891a2fef97070)) by [@vdusek](https://github.com/vdusek), closes [#396](https://github.com/apify/apify-sdk-python/issues/396), [#401](https://github.com/apify/apify-sdk-python/issues/401)
|
|
11
|
+
- Upgrade websockets to v14 to adapt to library API changes ([#425](https://github.com/apify/apify-sdk-python/pull/425)) ([5f49275](https://github.com/apify/apify-sdk-python/commit/5f49275ca1177e5ba56856ffe3860f6b97bee9ee)) by [@Mantisus](https://github.com/Mantisus), closes [#325](https://github.com/apify/apify-sdk-python/issues/325)
|
|
12
|
+
- Add signing of public URL ([#407](https://github.com/apify/apify-sdk-python/pull/407)) ([a865461](https://github.com/apify/apify-sdk-python/commit/a865461c703aea01d91317f4fdf38c1bedd35f00)) by [@danpoletaev](https://github.com/danpoletaev)
|
|
13
|
+
|
|
14
|
+
|
|
5
15
|
## [2.3.1](https://github.com/apify/apify-sdk-python/releases/tag/v2.3.1) (2025-02-25)
|
|
6
16
|
|
|
7
17
|
### 🐛 Bug Fixes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apify
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: Apify SDK for Python
|
|
5
5
|
Project-URL: Homepage, https://docs.apify.com/sdk/python/
|
|
6
6
|
Project-URL: Apify homepage, https://apify.com
|
|
@@ -224,14 +224,14 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
224
224
|
Classifier: Topic :: Software Development :: Libraries
|
|
225
225
|
Requires-Python: >=3.9
|
|
226
226
|
Requires-Dist: apify-client>=1.9.2
|
|
227
|
-
Requires-Dist: apify-shared>=1.
|
|
228
|
-
Requires-Dist: crawlee~=0.
|
|
227
|
+
Requires-Dist: apify-shared>=1.3.0
|
|
228
|
+
Requires-Dist: crawlee~=0.6.0
|
|
229
229
|
Requires-Dist: cryptography>=42.0.0
|
|
230
230
|
Requires-Dist: httpx>=0.27.0
|
|
231
231
|
Requires-Dist: lazy-object-proxy>=1.10.0
|
|
232
232
|
Requires-Dist: more-itertools>=10.2.0
|
|
233
233
|
Requires-Dist: typing-extensions>=4.1.0
|
|
234
|
-
Requires-Dist: websockets
|
|
234
|
+
Requires-Dist: websockets>=14.0
|
|
235
235
|
Provides-Extra: scrapy
|
|
236
236
|
Requires-Dist: scrapy>=2.11.0; extra == 'scrapy'
|
|
237
237
|
Description-Content-Type: text/markdown
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
---
|
|
2
|
+
id: pay-per-event
|
|
3
|
+
title: Pay-per-event monetization
|
|
4
|
+
description: Monetize your Actors using the pay-per-event pricing model
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
import ActorChargeSource from '!!raw-loader!./code/actor_charge.py';
|
|
8
|
+
import ConditionalActorChargeSource from '!!raw-loader!./code/conditional_actor_charge.py';
|
|
9
|
+
import ApiLink from '@site/src/components/ApiLink';
|
|
10
|
+
import CodeBlock from '@theme/CodeBlock';
|
|
11
|
+
|
|
12
|
+
Apify provides several [pricing models](https://docs.apify.com/platform/actors/publishing/monetize) for monetizing your Actors. The most recent and most flexible one is [pay-per-event](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event), which lets you charge your users programmatically directly from your Actor. As the name suggests, you may charge the users each time a specific event occurs, for example a call to an external API or when you return a result.
|
|
13
|
+
|
|
14
|
+
To use the pay-per-event pricing model, you first need to [set it up](https://docs.apify.com/platform/actors/running/actors-in-store#pay-per-event) for your Actor in the Apify console. After that, you're free to start charging for events.
|
|
15
|
+
|
|
16
|
+
## Charging for events
|
|
17
|
+
|
|
18
|
+
After monetization is set in the Apify console, you can add <ApiLink to="class/Actor#charge">`Actor.charge`</ApiLink> calls to your code and start monetizing!
|
|
19
|
+
|
|
20
|
+
<CodeBlock language="python">
|
|
21
|
+
{ActorChargeSource}
|
|
22
|
+
</CodeBlock>
|
|
23
|
+
|
|
24
|
+
Then you just push your code to Apify and that's it! The SDK will even keep track of the max total charge setting for you, so you will not provide more value than what the user chose to pay for.
|
|
25
|
+
|
|
26
|
+
If you need finer control over charging, you can access call <ApiLink to="class/Actor#get_charging_manager">`Actor.get_charging_manager()`</ApiLink> to access the <ApiLink to="class/ChargingManager">`ChargingManager`</ApiLink>, which can provide more detailed information - for example how many events of each type can be charged before reaching the configured limit.
|
|
27
|
+
|
|
28
|
+
## Transitioning from a different pricing model
|
|
29
|
+
|
|
30
|
+
When you plan to start using the pay-per-event pricing model for an Actor that is already monetized with a different pricing model, your source code will need support both pricing models during the transition period enforced by the Apify platform. Arguably the most frequent case is the transition from the pay-per-result model which utilizes the `ACTOR_MAX_PAID_DATASET_ITEMS` environment variable to prevent returning unpaid dataset items. The following is an example how to handle such scenarios. The key part is the <ApiLink to="class/ChargingManager#get_pricing_info">`ChargingManager.get_pricing_info()`</ApiLink> method which returns information about the current pricing model.
|
|
31
|
+
|
|
32
|
+
<CodeBlock language="python">
|
|
33
|
+
{ConditionalActorChargeSource}
|
|
34
|
+
</CodeBlock>
|
|
35
|
+
|
|
36
|
+
## Local development
|
|
37
|
+
|
|
38
|
+
It is encouraged to test your monetization code on your machine before releasing it to the public. To tell your Actor that it should work in pay-per-event mode, pass it the `ACTOR_TEST_PAY_PER_EVENT` environment variable:
|
|
39
|
+
|
|
40
|
+
```shell
|
|
41
|
+
ACTOR_TEST_PAY_PER_EVENT=true python -m youractor
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
If you also wish to see a log of all the events charged throughout the run, the Apify SDK keeps a log of charged events in a so called charging dataset. Your charging dataset can be found under the `charging_log` name (unless you change your storage settings, this dataset is stored in `storage/datasets/charging_log/`). Please note that this log is not available when running the Actor in production on the Apify platform.
|
|
45
|
+
|
|
46
|
+
Because pricing configuration is stored by the Apify platform, all events will have a default price of $1.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from apify import Actor
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
async def main() -> None:
|
|
5
|
+
async with Actor:
|
|
6
|
+
# highlight-start
|
|
7
|
+
# Charge for a single occurence of an event
|
|
8
|
+
await Actor.charge(event_name='init')
|
|
9
|
+
# highlight-end
|
|
10
|
+
|
|
11
|
+
# Prepare some mock results
|
|
12
|
+
result = [
|
|
13
|
+
{'word': 'Lorem'},
|
|
14
|
+
{'word': 'Ipsum'},
|
|
15
|
+
{'word': 'Dolor'},
|
|
16
|
+
{'word': 'Sit'},
|
|
17
|
+
{'word': 'Amet'},
|
|
18
|
+
]
|
|
19
|
+
# highlight-start
|
|
20
|
+
# Shortcut for charging for each pushed dataset item
|
|
21
|
+
await Actor.push_data(result, 'result-item')
|
|
22
|
+
# highlight-end
|
|
23
|
+
|
|
24
|
+
# highlight-start
|
|
25
|
+
# Or you can charge for a given number of events manually
|
|
26
|
+
await Actor.charge(
|
|
27
|
+
event_name='result-item',
|
|
28
|
+
count=len(result),
|
|
29
|
+
)
|
|
30
|
+
# highlight-end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from apify import Actor
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
async def main() -> None:
|
|
5
|
+
async with Actor:
|
|
6
|
+
# Check the dataset because there might already be items
|
|
7
|
+
# if the run migrated or was restarted
|
|
8
|
+
default_dataset = await Actor.open_dataset()
|
|
9
|
+
dataset_info = await default_dataset.get_info()
|
|
10
|
+
charged_items = dataset_info.item_count if dataset_info else 0
|
|
11
|
+
|
|
12
|
+
# highlight-start
|
|
13
|
+
if Actor.get_charging_manager().get_pricing_info().is_pay_per_event:
|
|
14
|
+
# highlight-end
|
|
15
|
+
await Actor.push_data({'hello': 'world'}, 'dataset-item')
|
|
16
|
+
elif charged_items < (Actor.config.max_paid_dataset_items or 0):
|
|
17
|
+
await Actor.push_data({'hello': 'world'})
|
|
18
|
+
charged_items += 1
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "apify"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.4.0"
|
|
8
8
|
description = "Apify SDK for Python"
|
|
9
9
|
authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -35,16 +35,14 @@ keywords = [
|
|
|
35
35
|
]
|
|
36
36
|
dependencies = [
|
|
37
37
|
"apify-client>=1.9.2",
|
|
38
|
-
"apify-shared>=1.
|
|
39
|
-
"crawlee~=0.
|
|
38
|
+
"apify-shared>=1.3.0",
|
|
39
|
+
"crawlee~=0.6.0",
|
|
40
40
|
"cryptography>=42.0.0",
|
|
41
41
|
"httpx>=0.27.0",
|
|
42
42
|
"lazy-object-proxy>=1.10.0",
|
|
43
43
|
"more_itertools>=10.2.0",
|
|
44
44
|
"typing-extensions>=4.1.0",
|
|
45
|
-
|
|
46
|
-
# https://github.com/apify/apify-sdk-python/issues/325
|
|
47
|
-
"websockets>=10.0,<14.0.0",
|
|
45
|
+
"websockets>=14.0",
|
|
48
46
|
]
|
|
49
47
|
|
|
50
48
|
[project.optional-dependencies]
|
|
@@ -62,7 +60,7 @@ scrapy = ["scrapy>=2.11.0"]
|
|
|
62
60
|
dev = [
|
|
63
61
|
"build~=1.2.0",
|
|
64
62
|
"filelock~=3.17.0",
|
|
65
|
-
"griffe~=1.
|
|
63
|
+
"griffe~=1.6.0",
|
|
66
64
|
"mypy~=1.15.0",
|
|
67
65
|
"pre-commit~=4.1.0",
|
|
68
66
|
"pydoc-markdown~=4.8.0",
|
|
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import os
|
|
5
5
|
import sys
|
|
6
|
+
from contextlib import suppress
|
|
6
7
|
from datetime import timedelta
|
|
7
8
|
from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, overload
|
|
8
9
|
|
|
@@ -43,7 +44,7 @@ if TYPE_CHECKING:
|
|
|
43
44
|
from typing_extensions import Self
|
|
44
45
|
|
|
45
46
|
from crawlee.proxy_configuration import _NewUrlFunction
|
|
46
|
-
from crawlee.storage_clients import
|
|
47
|
+
from crawlee.storage_clients import StorageClient
|
|
47
48
|
|
|
48
49
|
from apify._models import Webhook
|
|
49
50
|
|
|
@@ -64,6 +65,7 @@ class _ActorType:
|
|
|
64
65
|
configuration: Configuration | None = None,
|
|
65
66
|
*,
|
|
66
67
|
configure_logging: bool = True,
|
|
68
|
+
exit_process: bool | None = None,
|
|
67
69
|
) -> None:
|
|
68
70
|
"""Create an Actor instance.
|
|
69
71
|
|
|
@@ -74,7 +76,10 @@ class _ActorType:
|
|
|
74
76
|
configuration: The Actor configuration to be used. If not passed, a new Configuration instance will
|
|
75
77
|
be created.
|
|
76
78
|
configure_logging: Should the default logging configuration be configured?
|
|
79
|
+
exit_process: Whether the Actor should call `sys.exit` when the context manager exits. The default is
|
|
80
|
+
True except for the IPython, Pytest and Scrapy environments.
|
|
77
81
|
"""
|
|
82
|
+
self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process
|
|
78
83
|
self._is_exiting = False
|
|
79
84
|
|
|
80
85
|
self._configuration = configuration or Configuration.get_global_configuration()
|
|
@@ -141,9 +146,19 @@ class _ActorType:
|
|
|
141
146
|
|
|
142
147
|
return super().__repr__()
|
|
143
148
|
|
|
144
|
-
def __call__(
|
|
149
|
+
def __call__(
|
|
150
|
+
self,
|
|
151
|
+
configuration: Configuration | None = None,
|
|
152
|
+
*,
|
|
153
|
+
configure_logging: bool = True,
|
|
154
|
+
exit_process: bool | None = None,
|
|
155
|
+
) -> Self:
|
|
145
156
|
"""Make a new Actor instance with a non-default configuration."""
|
|
146
|
-
return self.__class__(
|
|
157
|
+
return self.__class__(
|
|
158
|
+
configuration=configuration,
|
|
159
|
+
configure_logging=configure_logging,
|
|
160
|
+
exit_process=exit_process,
|
|
161
|
+
)
|
|
147
162
|
|
|
148
163
|
@property
|
|
149
164
|
def apify_client(self) -> ApifyClientAsync:
|
|
@@ -171,7 +186,7 @@ class _ActorType:
|
|
|
171
186
|
return logger
|
|
172
187
|
|
|
173
188
|
@property
|
|
174
|
-
def _local_storage_client(self) ->
|
|
189
|
+
def _local_storage_client(self) -> StorageClient:
|
|
175
190
|
"""The local storage client the Actor instance uses."""
|
|
176
191
|
return service_locator.get_storage_client()
|
|
177
192
|
|
|
@@ -281,13 +296,7 @@ class _ActorType:
|
|
|
281
296
|
await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
|
|
282
297
|
self._is_initialized = False
|
|
283
298
|
|
|
284
|
-
if
|
|
285
|
-
self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in IPython')
|
|
286
|
-
elif os.getenv('PYTEST_CURRENT_TEST', default=False): # noqa: PLW1508
|
|
287
|
-
self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in an unit test')
|
|
288
|
-
elif os.getenv('SCRAPY_SETTINGS_MODULE'):
|
|
289
|
-
self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running with Scrapy')
|
|
290
|
-
else:
|
|
299
|
+
if self._exit_process:
|
|
291
300
|
sys.exit(exit_code)
|
|
292
301
|
|
|
293
302
|
async def fail(
|
|
@@ -1128,6 +1137,26 @@ class _ActorType:
|
|
|
1128
1137
|
|
|
1129
1138
|
return proxy_configuration
|
|
1130
1139
|
|
|
1140
|
+
def _get_default_exit_process(self) -> bool:
|
|
1141
|
+
"""Returns False for IPython, Pytest, and Scrapy environments, True otherwise."""
|
|
1142
|
+
if is_running_in_ipython():
|
|
1143
|
+
self.log.debug('Running in IPython, setting default `exit_process` to False.')
|
|
1144
|
+
return False
|
|
1145
|
+
|
|
1146
|
+
# Check if running in Pytest by detecting the relevant environment variable.
|
|
1147
|
+
if os.getenv('PYTEST_CURRENT_TEST'):
|
|
1148
|
+
self.log.debug('Running in Pytest, setting default `exit_process` to False.')
|
|
1149
|
+
return False
|
|
1150
|
+
|
|
1151
|
+
# Check if running in Scrapy by attempting to import it.
|
|
1152
|
+
with suppress(ImportError):
|
|
1153
|
+
import scrapy # noqa: F401
|
|
1154
|
+
|
|
1155
|
+
self.log.debug('Running in Scrapy, setting default `exit_process` to False.')
|
|
1156
|
+
return False
|
|
1157
|
+
|
|
1158
|
+
return True
|
|
1159
|
+
|
|
1131
1160
|
|
|
1132
1161
|
Actor = cast(_ActorType, Proxy(_ActorType))
|
|
1133
1162
|
"""The entry point of the SDK, through which all the Actor operations should be done."""
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import hmac
|
|
6
|
+
import string
|
|
4
7
|
from typing import Any
|
|
5
8
|
|
|
6
9
|
from cryptography.exceptions import InvalidTag as InvalidTagException
|
|
@@ -153,3 +156,38 @@ def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> An
|
|
|
153
156
|
)
|
|
154
157
|
|
|
155
158
|
return input_data
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
CHARSET = string.digits + string.ascii_letters
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def encode_base62(num: int) -> str:
|
|
165
|
+
"""Encode the given number to base62."""
|
|
166
|
+
if num == 0:
|
|
167
|
+
return CHARSET[0]
|
|
168
|
+
|
|
169
|
+
res = ''
|
|
170
|
+
while num > 0:
|
|
171
|
+
num, remainder = divmod(num, 62)
|
|
172
|
+
res = CHARSET[remainder] + res
|
|
173
|
+
return res
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@ignore_docs
|
|
177
|
+
def create_hmac_signature(secret_key: str, message: str) -> str:
|
|
178
|
+
"""Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.
|
|
179
|
+
|
|
180
|
+
HMAC signature is truncated to 30 characters to make it shorter.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
secret_key: Secret key used for signing signatures.
|
|
184
|
+
message: Message to be signed.
|
|
185
|
+
|
|
186
|
+
Returns:
|
|
187
|
+
Base62 encoded signature.
|
|
188
|
+
"""
|
|
189
|
+
signature = hmac.new(secret_key.encode('utf-8'), message.encode('utf-8'), hashlib.sha256).hexdigest()[:30]
|
|
190
|
+
|
|
191
|
+
decimal_signature = int(signature, 16)
|
|
192
|
+
|
|
193
|
+
return encode_base62(decimal_signature)
|
|
@@ -4,7 +4,7 @@ import asyncio
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from typing import TYPE_CHECKING, Annotated, Any, Literal, Union
|
|
6
6
|
|
|
7
|
-
import websockets.client
|
|
7
|
+
import websockets.asyncio.client
|
|
8
8
|
from pydantic import BaseModel, Discriminator, Field, TypeAdapter
|
|
9
9
|
from typing_extensions import Self, Unpack, override
|
|
10
10
|
|
|
@@ -143,7 +143,7 @@ class PlatformEventManager(EventManager):
|
|
|
143
143
|
but instead use it via the `Actor.on()` and `Actor.off()` methods.
|
|
144
144
|
"""
|
|
145
145
|
|
|
146
|
-
_platform_events_websocket: websockets.client.
|
|
146
|
+
_platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
|
|
147
147
|
_process_platform_messages_task: asyncio.Task | None = None
|
|
148
148
|
_send_system_info_interval_task: asyncio.Task | None = None
|
|
149
149
|
_connected_to_platform_websocket: asyncio.Future = asyncio.Future()
|
|
@@ -196,7 +196,7 @@ class PlatformEventManager(EventManager):
|
|
|
196
196
|
|
|
197
197
|
async def _process_platform_messages(self, ws_url: str) -> None:
|
|
198
198
|
try:
|
|
199
|
-
async with websockets.client.connect(ws_url) as websocket:
|
|
199
|
+
async with websockets.asyncio.client.connect(ws_url) as websocket:
|
|
200
200
|
self._platform_events_websocket = websocket
|
|
201
201
|
self._connected_to_platform_websocket.set_result(True)
|
|
202
202
|
|
|
@@ -6,7 +6,7 @@ from typing_extensions import override
|
|
|
6
6
|
|
|
7
7
|
from apify_client import ApifyClientAsync
|
|
8
8
|
from crawlee._utils.crypto import crypto_random_object_id
|
|
9
|
-
from crawlee.storage_clients import
|
|
9
|
+
from crawlee.storage_clients import StorageClient
|
|
10
10
|
|
|
11
11
|
from apify._utils import docs_group
|
|
12
12
|
from apify.apify_storage_client._dataset_client import DatasetClient
|
|
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
@docs_group('Classes')
|
|
24
|
-
class ApifyStorageClient(
|
|
24
|
+
class ApifyStorageClient(StorageClient):
|
|
25
25
|
"""A storage client implementation based on the Apify platform storage."""
|
|
26
26
|
|
|
27
27
|
def __init__(self, *, configuration: Configuration) -> None:
|
|
@@ -68,5 +68,5 @@ class ApifyStorageClient(BaseStorageClient):
|
|
|
68
68
|
pass
|
|
69
69
|
|
|
70
70
|
@override
|
|
71
|
-
def get_rate_limit_errors(self) -> dict[int, int]:
|
|
71
|
+
def get_rate_limit_errors(self) -> dict[int, int]:
|
|
72
72
|
return self._apify_client.stats.rate_limit_errors
|
|
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
|
|
|
4
4
|
|
|
5
5
|
from typing_extensions import override
|
|
6
6
|
|
|
7
|
-
from crawlee.storage_clients._base import BaseDatasetClient
|
|
7
|
+
from crawlee.storage_clients._base import DatasetClient as BaseDatasetClient
|
|
8
8
|
from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
|
|
|
4
4
|
|
|
5
5
|
from typing_extensions import override
|
|
6
6
|
|
|
7
|
-
from crawlee.storage_clients._base import BaseDatasetCollectionClient
|
|
7
|
+
from crawlee.storage_clients._base import DatasetCollectionClient as BaseDatasetCollectionClient
|
|
8
8
|
from crawlee.storage_clients.models import DatasetListPage, DatasetMetadata
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
@@ -4,10 +4,13 @@ from contextlib import asynccontextmanager
|
|
|
4
4
|
from typing import TYPE_CHECKING, Any
|
|
5
5
|
|
|
6
6
|
from typing_extensions import override
|
|
7
|
+
from yarl import URL
|
|
7
8
|
|
|
8
|
-
from crawlee.storage_clients._base import BaseKeyValueStoreClient
|
|
9
|
+
from crawlee.storage_clients._base import KeyValueStoreClient as BaseKeyValueStoreClient
|
|
9
10
|
from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
|
|
10
11
|
|
|
12
|
+
from apify._crypto import create_hmac_signature
|
|
13
|
+
|
|
11
14
|
if TYPE_CHECKING:
|
|
12
15
|
from collections.abc import AsyncIterator
|
|
13
16
|
from contextlib import AbstractAsyncContextManager
|
|
@@ -89,6 +92,18 @@ class KeyValueStoreClient(BaseKeyValueStoreClient):
|
|
|
89
92
|
Args:
|
|
90
93
|
key: The key for which the URL should be generated.
|
|
91
94
|
"""
|
|
92
|
-
|
|
95
|
+
if self._client.resource_id is None:
|
|
96
|
+
raise ValueError('resource_id cannot be None when generating a public URL')
|
|
97
|
+
|
|
98
|
+
public_url = (
|
|
99
|
+
URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._client.resource_id / 'records' / key
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
key_value_store = await self.get()
|
|
103
|
+
|
|
104
|
+
if key_value_store is not None and isinstance(key_value_store.model_extra, dict):
|
|
105
|
+
url_signing_secret_key = key_value_store.model_extra.get('urlSigningSecretKey')
|
|
106
|
+
if url_signing_secret_key:
|
|
107
|
+
public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
|
|
93
108
|
|
|
94
|
-
return
|
|
109
|
+
return str(public_url)
|
{apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_key_value_store_collection_client.py
RENAMED
|
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
|
|
|
4
4
|
|
|
5
5
|
from typing_extensions import override
|
|
6
6
|
|
|
7
|
-
from crawlee.storage_clients._base import BaseKeyValueStoreCollectionClient
|
|
7
|
+
from crawlee.storage_clients._base import KeyValueStoreCollectionClient as BaseKeyValueStoreCollectionClient
|
|
8
8
|
from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING
|
|
|
5
5
|
from typing_extensions import override
|
|
6
6
|
|
|
7
7
|
from crawlee import Request
|
|
8
|
-
from crawlee.storage_clients._base import BaseRequestQueueClient
|
|
8
|
+
from crawlee.storage_clients._base import RequestQueueClient as BaseRequestQueueClient
|
|
9
9
|
from crawlee.storage_clients.models import (
|
|
10
10
|
BatchRequestsOperationResponse,
|
|
11
11
|
ProcessedRequest,
|
{apify-2.3.1b2 → apify-2.4.0}/src/apify/apify_storage_client/_request_queue_collection_client.py
RENAMED
|
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
|
|
|
4
4
|
|
|
5
5
|
from typing_extensions import override
|
|
6
6
|
|
|
7
|
-
from crawlee.storage_clients._base import BaseRequestQueueCollectionClient
|
|
7
|
+
from crawlee.storage_clients._base import RequestQueueCollectionClient as BaseRequestQueueCollectionClient
|
|
8
8
|
from crawlee.storage_clients.models import RequestQueueListPage, RequestQueueMetadata
|
|
9
9
|
|
|
10
10
|
if TYPE_CHECKING:
|
|
@@ -10,7 +10,7 @@ from pydantic import BaseModel, Field, TypeAdapter
|
|
|
10
10
|
|
|
11
11
|
from crawlee import Request
|
|
12
12
|
from crawlee._types import HttpMethod
|
|
13
|
-
from crawlee.http_clients import
|
|
13
|
+
from crawlee.http_clients import HttpClient, HttpxHttpClient
|
|
14
14
|
from crawlee.request_loaders import RequestList as CrawleeRequestList
|
|
15
15
|
|
|
16
16
|
from apify._utils import docs_group
|
|
@@ -49,7 +49,7 @@ class RequestList(CrawleeRequestList):
|
|
|
49
49
|
async def open(
|
|
50
50
|
name: str | None = None,
|
|
51
51
|
request_list_sources_input: list[dict[str, Any]] | None = None,
|
|
52
|
-
http_client:
|
|
52
|
+
http_client: HttpClient | None = None,
|
|
53
53
|
) -> RequestList:
|
|
54
54
|
"""Creates RequestList from Actor input requestListSources.
|
|
55
55
|
|
|
@@ -78,7 +78,7 @@ class RequestList(CrawleeRequestList):
|
|
|
78
78
|
|
|
79
79
|
@staticmethod
|
|
80
80
|
async def _create_request_list(
|
|
81
|
-
name: str | None, request_list_sources_input: list[dict[str, Any]], http_client:
|
|
81
|
+
name: str | None, request_list_sources_input: list[dict[str, Any]], http_client: HttpClient | None
|
|
82
82
|
) -> RequestList:
|
|
83
83
|
if not http_client:
|
|
84
84
|
http_client = HttpxHttpClient()
|
|
@@ -108,7 +108,7 @@ class RequestList(CrawleeRequestList):
|
|
|
108
108
|
|
|
109
109
|
@staticmethod
|
|
110
110
|
async def _fetch_requests_from_url(
|
|
111
|
-
remote_url_requests_inputs: list[_RequestsFromUrlInput], http_client:
|
|
111
|
+
remote_url_requests_inputs: list[_RequestsFromUrlInput], http_client: HttpClient
|
|
112
112
|
) -> list[Request]:
|
|
113
113
|
"""Crete list of requests from url.
|
|
114
114
|
|
|
@@ -201,19 +201,28 @@ async def test_generate_public_url_for_kvs_record(
|
|
|
201
201
|
run_actor: RunActorFunction,
|
|
202
202
|
) -> None:
|
|
203
203
|
async def main() -> None:
|
|
204
|
-
from
|
|
205
|
-
|
|
206
|
-
from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient
|
|
204
|
+
from apify._crypto import create_hmac_signature
|
|
207
205
|
|
|
208
206
|
async with Actor:
|
|
209
207
|
public_api_url = Actor.config.api_public_base_url
|
|
210
208
|
default_store_id = Actor.config.default_key_value_store_id
|
|
209
|
+
record_key = 'public-record-key'
|
|
211
210
|
|
|
212
211
|
store = await Actor.open_key_value_store()
|
|
213
|
-
record_url = await cast(KeyValueStoreClient, store._resource_client).get_public_url('dummy')
|
|
214
|
-
print(record_url)
|
|
215
212
|
|
|
216
|
-
assert
|
|
213
|
+
assert isinstance(store.storage_object.model_extra, dict)
|
|
214
|
+
url_signing_secret_key = store.storage_object.model_extra.get('urlSigningSecretKey')
|
|
215
|
+
assert url_signing_secret_key is not None
|
|
216
|
+
|
|
217
|
+
await store.set_value(record_key, {'exposedData': 'test'}, 'application/json')
|
|
218
|
+
|
|
219
|
+
record_url = await store.get_public_url(record_key)
|
|
220
|
+
|
|
221
|
+
signature = create_hmac_signature(url_signing_secret_key, record_key)
|
|
222
|
+
assert (
|
|
223
|
+
record_url
|
|
224
|
+
== f'{public_api_url}/v2/key-value-stores/{default_store_id}/records/{record_key}?signature={signature}'
|
|
225
|
+
)
|
|
217
226
|
|
|
218
227
|
actor = await make_actor(label='kvs-get-public-url', main_func=main)
|
|
219
228
|
run_result = await run_actor(actor)
|
|
@@ -2,12 +2,17 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
4
|
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
5
7
|
from apify import Actor, __version__
|
|
6
8
|
|
|
7
9
|
if TYPE_CHECKING:
|
|
8
10
|
from .conftest import MakeActorFunction, RunActorFunction
|
|
9
11
|
|
|
10
12
|
|
|
13
|
+
# TODO: What to do with the `browserforge` output?
|
|
14
|
+
# https://github.com/apify/apify-sdk-python/issues/423
|
|
15
|
+
@pytest.mark.skip
|
|
11
16
|
async def test_actor_logging(
|
|
12
17
|
make_actor: MakeActorFunction,
|
|
13
18
|
run_actor: RunActorFunction,
|
|
@@ -9,7 +9,7 @@ from typing import Any, Callable, cast
|
|
|
9
9
|
from unittest.mock import AsyncMock, Mock
|
|
10
10
|
|
|
11
11
|
import pytest
|
|
12
|
-
import websockets.server
|
|
12
|
+
import websockets.asyncio.server
|
|
13
13
|
from lazy_object_proxy import Proxy
|
|
14
14
|
|
|
15
15
|
from apify_shared.consts import ActorEnvVars, ApifyEnvVars
|
|
@@ -139,10 +139,10 @@ async def test_actor_handles_migrating_event_correctly(monkeypatch: pytest.Monke
|
|
|
139
139
|
nonlocal persist_state_events_data
|
|
140
140
|
persist_state_events_data.append(data)
|
|
141
141
|
|
|
142
|
-
async def handler(websocket: websockets.server.
|
|
142
|
+
async def handler(websocket: websockets.asyncio.server.ServerConnection) -> None:
|
|
143
143
|
await websocket.wait_closed()
|
|
144
144
|
|
|
145
|
-
async with websockets.server.serve(handler, host='localhost') as ws_server:
|
|
145
|
+
async with websockets.asyncio.server.serve(handler, host='localhost') as ws_server:
|
|
146
146
|
port: int = ws_server.sockets[0].getsockname()[1] # type: ignore[index]
|
|
147
147
|
monkeypatch.setenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL, f'ws://localhost:{port}')
|
|
148
148
|
|
|
@@ -181,7 +181,7 @@ async def test_actor_handles_migrating_event_correctly(monkeypatch: pytest.Monke
|
|
|
181
181
|
Actor.on(Event.PERSIST_STATE, log_persist_state)
|
|
182
182
|
await asyncio.sleep(2)
|
|
183
183
|
|
|
184
|
-
for socket in ws_server.
|
|
184
|
+
for socket in ws_server.connections:
|
|
185
185
|
await socket.send(
|
|
186
186
|
json.dumps(
|
|
187
187
|
{
|