apify 2.7.1b6__tar.gz → 2.7.1b7__tar.gz

{apify-2.7.1b6 → apify-2.7.1b7}/.github/workflows/build_and_deploy_docs.yaml

@@ -22,7 +22,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
 

{apify-2.7.1b6 → apify-2.7.1b7}/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### 🐛 Bug Fixes
 
 - Restrict apify-shared and apify-client versions ([#523](https://github.com/apify/apify-sdk-python/pull/523)) ([b3ae5a9](https://github.com/apify/apify-sdk-python/commit/b3ae5a972a65454a4998eda59c9fcc3f6b7e8579)) by [@vdusek](https://github.com/vdusek)
+- Expose `APIFY_USER_IS_PAYING` env var to the configuration ([#507](https://github.com/apify/apify-sdk-python/pull/507)) ([0801e54](https://github.com/apify/apify-sdk-python/commit/0801e54887317c1280cc6828ecd3f2cc53287e76)) by [@stepskop](https://github.com/stepskop)
 
 
 <!-- git-cliff-unreleased-end -->
@@ -23,16 +24,11 @@ All notable changes to this project will be documented in this file.
 - Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
 
 
-## [2.7.0](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.0) (2025-07-14)
-
-### 🚀 Features
-
-- **crypto:** Decrypt secret objects ([#482](https://github.com/apify/apify-sdk-python/pull/482)) ([ce9daf7](https://github.com/apify/apify-sdk-python/commit/ce9daf7381212b8dc194e8a643e5ca0dedbc0078)) by [@MFori](https://github.com/MFori)
+## [2.7.3](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.3) (2025-08-11)
 
 ### 🐛 Bug Fixes
 
-- Sync `@docusaurus` theme version [internal] ([#500](https://github.com/apify/apify-sdk-python/pull/500)) ([a7485e7](https://github.com/apify/apify-sdk-python/commit/a7485e7d2276fde464ce862573d5b95e7d4d836a)) by [@katzino](https://github.com/katzino)
-- Tagline overlap ([#501](https://github.com/apify/apify-sdk-python/pull/501)) ([bae8340](https://github.com/apify/apify-sdk-python/commit/bae8340c46fea756ea35ea4d591da84c09d478e2)) by [@katzino](https://github.com/katzino)
+- Expose `APIFY_USER_IS_PAYING` env var to the configuration (#507) ([0de022c](https://github.com/apify/apify-sdk-python/commit/0de022c3435f24c821053c771e7b659433e3fb6e))
 
 
 ## [2.7.2](https://github.com/apify/apify-sdk-python/releases/tag/v2.7.2) (2025-07-30)

{apify-2.7.1b6 → apify-2.7.1b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 2.7.1b6
+Version: 2.7.1b7
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog

{apify-2.7.1b6 → apify-2.7.1b7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "2.7.1b6"
+version = "2.7.1b7"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -65,19 +65,21 @@ dev = [
     "build~=1.3.0",
     "crawlee[parsel]",
     "dycw-pytest-only>=2.1.1",
-    "griffe~=1.9.0",
+    "griffe~=1.11.0",
     "mypy~=1.17.0",
-    "pre-commit~=4.2.0",
+    "pre-commit~=4.3.0",
     "pydoc-markdown~=4.8.0",
     "pytest-asyncio~=1.1.0",
     "pytest-cov~=6.2.0",
+    "pytest-httpserver>=1.1.3",
     "pytest-timeout>=2.4.0",
     "pytest-xdist~=3.8.0",
     "pytest~=8.4.0",
-    "respx~=0.22.0",
     "ruff~=0.12.0",
     "setuptools", # setuptools are used by pytest but not explicitly required
     "uvicorn[standard]",
+    "werkzeug~=3.1.3", # Werkzeug is used by httpserver
+    "yarl~=1.20.0", # yarl is used by crawlee
 ]
 
 [tool.hatch.build.targets.wheel]

{apify-2.7.1b6 → apify-2.7.1b7}/src/apify/_configuration.py

@@ -334,6 +334,15 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    user_is_paying: Annotated[
+        bool,
+        Field(
+            alias='apify_user_is_paying',
+            description='True if the user calling the Actor is paying user',
+        ),
+        BeforeValidator(lambda val: False if val == '' else val),
+    ] = False
+
     web_server_port: Annotated[
         int,
         Field(

{apify-2.7.1b6 → apify-2.7.1b7}/tests/integration/test_crawlers_with_storages.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+import pytest
+
 if TYPE_CHECKING:
     from .conftest import MakeActorFunction, RunActorFunction
 
@@ -76,6 +78,7 @@ async def test_actor_on_platform_max_requests_per_crawl(
     assert run_result.status == 'SUCCEEDED'
 
 
+@pytest.mark.skip(reason='Sometimes crawler does not respect max_request_retries argument, see issue #540')
 async def test_actor_on_platform_max_request_retries(
     make_actor: MakeActorFunction,
     run_actor: RunActorFunction,

{apify-2.7.1b6 → apify-2.7.1b7}/tests/unit/actor/test_actor_create_proxy_configuration.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING
+from unittest.mock import Mock
 
-import httpx
 import pytest
 
 from apify_client import ApifyClientAsync
@@ -11,7 +11,8 @@ from apify_shared.consts import ApifyEnvVars
 from apify import Actor
 
 if TYPE_CHECKING:
-    from respx import MockRouter
+    from pytest_httpserver import HTTPServer
+    from werkzeug import Request, Response
 
     from ..conftest import ApifyClientAsyncPatcher
 
@@ -24,25 +25,29 @@ def patched_apify_client(apify_client_async_patcher: ApifyClientAsyncPatcher) ->
     return ApifyClientAsync()
 
 
+@pytest.mark.usefixtures('patched_httpx_client')
 async def test_basic_proxy_configuration_creation(
     monkeypatch: pytest.MonkeyPatch,
-    respx_mock: MockRouter,
+    httpserver: HTTPServer,
     patched_apify_client: ApifyClientAsync,
 ) -> None:
-    dummy_proxy_status_url = 'http://dummy-proxy-status-url.com'
+    dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/')
     monkeypatch.setenv(ApifyEnvVars.TOKEN.value, 'DUMMY_TOKEN')
     monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url)
 
-    route = respx_mock.get(dummy_proxy_status_url)
-    route.mock(
-        httpx.Response(
-            200,
-            json={
-                'connected': True,
-                'connectionError': None,
-                'isManInTheMiddle': True,
-            },
-        )
+    call_mock = Mock()
+
+    def request_handler(request: Request, response: Response) -> Response:
+        call_mock(request.url)
+        return response
+
+    httpserver.expect_oneshot_request('/').with_post_hook(request_handler).respond_with_json(
+        {
+            'connected': True,
+            'connectionError': None,
+            'isManInTheMiddle': True,
+        },
+        status=200,
     )
 
     groups = ['GROUP1', 'GROUP2']
@@ -58,32 +63,36 @@ async def test_basic_proxy_configuration_creation(
     assert proxy_configuration._country_code == country_code
 
     assert len(patched_apify_client.calls['user']['get']) == 1  # type: ignore[attr-defined]
-    assert len(route.calls) == 1
+    assert call_mock.call_count == 1
 
     await Actor.exit()
 
 
+@pytest.mark.usefixtures('patched_httpx_client')
 async def test_proxy_configuration_with_actor_proxy_input(
     monkeypatch: pytest.MonkeyPatch,
-    respx_mock: MockRouter,
+    httpserver: HTTPServer,
     patched_apify_client: ApifyClientAsync,
 ) -> None:
-    dummy_proxy_status_url = 'http://dummy-proxy-status-url.com'
+    dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/')
     dummy_proxy_url = 'http://dummy-proxy.com:8000'
 
     monkeypatch.setenv(ApifyEnvVars.TOKEN.value, 'DUMMY_TOKEN')
     monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url)
 
-    route = respx_mock.get(dummy_proxy_status_url)
-    route.mock(
-        httpx.Response(
-            200,
-            json={
-                'connected': True,
-                'connectionError': None,
-                'isManInTheMiddle': True,
-            },
-        )
+    call_mock = Mock()
+
+    def request_handler(request: Request, response: Response) -> Response:
+        call_mock(request.url)
+        return response
+
+    httpserver.expect_request('/').with_post_hook(request_handler).respond_with_json(
+        {
+            'connected': True,
+            'connectionError': None,
+            'isManInTheMiddle': True,
+        },
+        status=200,
     )
 
     await Actor.init()
@@ -138,6 +147,6 @@ async def test_proxy_configuration_with_actor_proxy_input(
     )
 
     assert len(patched_apify_client.calls['user']['get']) == 2  # type: ignore[attr-defined]
-    assert len(route.calls) == 2
+    assert call_mock.call_count == 2
 
     await Actor.exit()

{apify-2.7.1b6 → apify-2.7.1b7}/tests/unit/actor/test_request_list.py

@@ -2,17 +2,21 @@ from __future__ import annotations
 
 import re
 from dataclasses import dataclass
-from typing import Any, get_args
+from typing import TYPE_CHECKING, Any, get_args
+from unittest.mock import Mock
 
 import pytest
-import respx
-from httpx import Response
+from yarl import URL
 
 from crawlee._request import UserData
 from crawlee._types import HttpMethod
 
 from apify.storages._request_list import URL_NO_COMMAS_REGEX, RequestList
 
+if TYPE_CHECKING:
+    from pytest_httpserver import HTTPServer
+    from werkzeug import Request, Response
+
 
 @pytest.mark.parametrize(
     argnames='request_method',
@@ -67,20 +71,19 @@ async def test_request_list_open_request_types(
     assert request.headers.root == optional_input.get('headers', {})
 
 
-@respx.mock
-async def test_request_list_open_from_url_correctly_send_requests() -> None:
+async def test_request_list_open_from_url_correctly_send_requests(httpserver: HTTPServer) -> None:
     """Test that requests are sent to expected urls."""
     request_list_sources_input: list[dict[str, Any]] = [
         {
-            'requestsFromUrl': 'https://abc.dev/file.txt',
+            'requestsFromUrl': httpserver.url_for('/file.txt'),
             'method': 'GET',
         },
         {
-            'requestsFromUrl': 'https://www.abc.dev/file2',
+            'requestsFromUrl': httpserver.url_for('/file2'),
             'method': 'PUT',
         },
         {
-            'requestsFromUrl': 'https://www.something.som',
+            'requestsFromUrl': httpserver.url_for('/something'),
             'method': 'POST',
             'headers': {'key': 'value'},
             'payload': 'some_payload',
@@ -88,16 +91,28 @@ async def test_request_list_open_from_url_correctly_send_requests() -> None:
         },
     ]
 
-    routes = [respx.get(entry['requestsFromUrl']) for entry in request_list_sources_input]
+    routes: dict[str, Mock] = {}
+
+    def request_handler(request: Request, response: Response) -> Response:
+        routes[request.url]()
+        return response
+
+    for entry in request_list_sources_input:
+        path = str(URL(entry['requestsFromUrl']).path)
+        httpserver.expect_oneshot_request(path).with_post_hook(request_handler).respond_with_data(status=200)
+        routes[entry['requestsFromUrl']] = Mock()
 
     await RequestList.open(request_list_sources_input=request_list_sources_input)
 
-    for route in routes:
-        assert route.called
+    assert len(routes) == len(request_list_sources_input)
 
+    for entity in request_list_sources_input:
+        entity_url = entity['requestsFromUrl']
+        assert entity_url in routes
+        assert routes[entity_url].called
 
-@respx.mock
-async def test_request_list_open_from_url() -> None:
+
+async def test_request_list_open_from_url(httpserver: HTTPServer) -> None:
     """Test that create_request_list is correctly reading urls from remote url sources and also from simple input."""
     expected_simple_url = 'https://www.someurl.com'
     expected_remote_urls_1 = {'http://www.something.com', 'https://www.somethingelse.com', 'http://www.bla.net'}
@@ -111,11 +126,11 @@ async def test_request_list_open_from_url() -> None:
 
     mocked_urls = (
         MockedUrlInfo(
-            'https://abc.dev/file.txt',
+            httpserver.url_for('/file.txt'),
             'blablabla{} more blablabla{} , even more blablabla. {} '.format(*expected_remote_urls_1),
         ),
         MockedUrlInfo(
-            'https://www.abc.dev/file2',
+            httpserver.url_for('/file2'),
             'some stuff{} more stuff{} www.false_positive.com'.format(*expected_remote_urls_2),
         ),
     )
@@ -132,7 +147,8 @@ async def test_request_list_open_from_url() -> None:
         },
     ]
     for mocked_url in mocked_urls:
-        respx.get(mocked_url.url).mock(return_value=Response(200, text=mocked_url.response_text))
+        path = str(URL(mocked_url.url).path)
+        httpserver.expect_oneshot_request(path).respond_with_data(status=200, response_data=mocked_url.response_text)
 
     request_list = await RequestList.open(request_list_sources_input=request_list_sources_input)
     generated_requests = []
@@ -143,23 +159,20 @@ async def test_request_list_open_from_url() -> None:
     assert {generated_request.url for generated_request in generated_requests} == expected_urls
 
 
-@respx.mock
-async def test_request_list_open_from_url_additional_inputs() -> None:
+async def test_request_list_open_from_url_additional_inputs(httpserver: HTTPServer) -> None:
     """Test that all generated request properties are correctly populated from input values."""
     expected_url = 'https://www.someurl.com'
     example_start_url_input: dict[str, Any] = {
-        'requestsFromUrl': 'https://crawlee.dev/file.txt',
+        'requestsFromUrl': httpserver.url_for('/file.txt'),
         'method': 'POST',
         'headers': {'key': 'value'},
         'payload': 'some_payload',
         'userData': {'another_key': 'another_value'},
     }
-
-    respx.get(example_start_url_input['requestsFromUrl']).mock(return_value=Response(200, text=expected_url))
+    httpserver.expect_oneshot_request('/file.txt').respond_with_data(status=200, response_data=expected_url)
 
     request_list = await RequestList.open(request_list_sources_input=[example_start_url_input])
     request = await request_list.fetch_next_request()
-
     # Check all properties correctly created for request
     assert request
     assert request.url == expected_url

{apify-2.7.1b6 → apify-2.7.1b7}/tests/unit/conftest.py

@@ -4,9 +4,12 @@ import asyncio
 import inspect
 import os
 from collections import defaultdict
+from logging import getLogger
 from typing import TYPE_CHECKING, Any, get_type_hints
 
+import httpx
 import pytest
+from pytest_httpserver import HTTPServer
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ApifyEnvVars
@@ -18,7 +21,7 @@ from crawlee.storages import _creation_management
 import apify._actor
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
+    from collections.abc import Callable, Iterator
     from pathlib import Path
 
 
@@ -187,3 +190,37 @@ def memory_storage_client() -> MemoryStorageClient:
     configuration.write_metadata = True
 
     return MemoryStorageClient.from_config(configuration)
+
+
+@pytest.fixture(scope='session')
+def make_httpserver() -> Iterator[HTTPServer]:
+    werkzeug_logger = getLogger('werkzeug')
+    werkzeug_logger.disabled = True
+
+    server = HTTPServer(threaded=True, host='127.0.0.1')
+    server.start()
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+    if server.is_running():
+        server.stop()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def httpserver(make_httpserver: HTTPServer) -> Iterator[HTTPServer]:
+    server = make_httpserver
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def patched_httpx_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    """Patch httpx client to drop proxy settings."""
+
+    class ProxylessAsyncClient(httpx.AsyncClient):
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            kwargs.pop('proxy', None)
+            super().__init__(*args, **kwargs)
+
+    monkeypatch.setattr(httpx, 'AsyncClient', ProxylessAsyncClient)
+    yield
+    monkeypatch.undo()