scrapely-python 0.1.1__tar.gz → 0.1.3__tar.gz

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/PKG-INFO

@@ -1,16 +1,15 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.1
+Version: 0.1.3
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://scrape.ly
 Project-URL: Documentation, https://docs.scrape.ly
 Project-URL: Repository, https://github.com/SandeepDev1/scrapely
 Keywords: scraping,actor,serverless,apify,scrapely
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
@@ -22,7 +21,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: httpx>=0.25.0
 Requires-Dist: crawlee<2.0.0,>=1.8.0
 Requires-Dist: websockets>=12.0
-Requires-Dist: scrapely-client-python<2,>=1.0.0
+Requires-Dist: scrapely-client-python<2,>=1.0.2
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: cryptography>=42.0.0
 Provides-Extra: dev

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/pyproject.toml

@@ -1,20 +1,19 @@
 [build-system]
-requires = ["setuptools>=61.0", "wheel"]
+requires = ["setuptools>=77.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.1"
+version = "0.1.3"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
-license = {text = "MIT"}
+license = "MIT"
 authors = [
     {name = "Scrapely", email = "support@scrape.ly"}
 ]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
-    "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
@@ -28,7 +27,7 @@ dependencies = [
     "httpx>=0.25.0",
     "crawlee>=1.8.0,<2.0.0",
     "websockets>=12.0",
-    "scrapely-client-python>=1.0.0,<2",
+    "scrapely-client-python>=1.0.2,<2",
     "pydantic>=2.0.0",
     "cryptography>=42.0.0",
 ]

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.0'
+__version__ = '0.1.3'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely/actor.py

@@ -1776,19 +1776,20 @@ class Actor(metaclass=_ActorMeta):
         """
         cls._ensure_initialized('create_proxy_configuration')
 
-        from .proxy_configuration import ProxyConfiguration
+        from .proxy_configuration import ProxyConfiguration, _input_proxy_option, _input_use_platform_proxy
 
         # Handle actor_proxy_input (Node SDK normalizeProxyInput/createProxyConfiguration
-        # parity): `useApifyProxy: false` explicitly disables the PLATFORM proxy only —
+        # parity): `useScrapelyProxy: false` (or the legacy `useApifyProxy: false`; a
+        # boolean useScrapelyProxy wins) explicitly disables the PLATFORM proxy only —
         # custom proxies (proxyUrls / tiered lists / new_url_function) from the same
         # input must still be used. The standard "Custom proxy URLs" input shape is
-        # {useApifyProxy: false, proxyUrls: [...]}; returning None for it would
+        # {useScrapelyProxy: false, proxyUrls: [...]}; returning None for it would
         # silently drop the user's own proxies and crawl on their real IPs. We only
         # bail when no custom proxies are given either. A MISSING key falls back to
         # platform mode when the server injected credentials — input like
-        # {apifyProxyGroups: [...]} must not silently disable proxying.
+        # {scrapelyProxyGroups: [...]} must not silently disable proxying.
         if actor_proxy_input is not None:
-            if actor_proxy_input.get('useApifyProxy') is False and not (
+            if _input_use_platform_proxy(actor_proxy_input) is False and not (
                 proxy_urls
                 or new_url_function
                 or tiered_proxy_urls
@@ -1798,9 +1799,9 @@ class Actor(metaclass=_ActorMeta):
                 or actor_proxy_input.get('tieredProxyConfig')
             ):
                 return None
-            country_code = country_code or actor_proxy_input.get('apifyProxyCountry')
-            subdivision_code = subdivision_code or actor_proxy_input.get('apifyProxySubdivision')
-            groups = groups or actor_proxy_input.get('apifyProxyGroups')
+            country_code = country_code or _input_proxy_option(actor_proxy_input, 'Country')
+            subdivision_code = subdivision_code or _input_proxy_option(actor_proxy_input, 'Subdivision')
+            groups = groups or _input_proxy_option(actor_proxy_input, 'Groups')
             proxy_urls = proxy_urls or actor_proxy_input.get('proxyUrls')
             tiered_proxy_urls = tiered_proxy_urls or actor_proxy_input.get('tieredProxyUrls')
             tiered_proxy_config = tiered_proxy_config or actor_proxy_input.get('tieredProxyConfig')

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely/charging.py

@@ -195,6 +195,11 @@ class ChargingManager:
         self._pending_charge_keys: Dict[str, Dict[str, Any]] = {}
         # Optional local charging-log dataset (ACTOR_USE_CHARGING_LOG_DATASET).
         self._charging_log_dataset: Any = None
+        # Idempotency key of the last charge the platform accepted
+        # (push_data_and_charge voids it when the push then fails).
+        self._last_accepted_operation_id: Optional[str] = None
+        # Voids not yet confirmed by the platform; retried before the next push and at exit.
+        self._pending_voids: List[tuple[str, str, int]] = []
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -429,6 +434,50 @@ class ChargingManager:
                 self._total_charged_usd - event_price * Decimal(count),
             )
 
+    def _take_last_accepted_operation_id(self) -> Optional[str]:
+        op_id = self._last_accepted_operation_id
+        self._last_accepted_operation_id = None
+        return op_id
+
+    async def _void_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+        """Reverse an accepted charge whose items were never stored, then unwind locally.
+
+        POST /v2/actor-runs/{run}/charge/void with the charge's Idempotency-Key.
+        The local reservation is released only once the platform confirms.
+        """
+        if self._client is None or not self._is_at_home or not self._actor_run_id:
+            return False
+        for attempt in range(self._POST_ATTEMPTS):
+            try:
+                await self._client.post(
+                    f'/v2/actor-runs/{self._actor_run_id}/charge/void',
+                    json_data={},
+                    headers={'Idempotency-Key': operation_id},
+                )
+                self._unwind_reservation(event_name, reserved)
+                return True
+            except Exception as err:  # noqa: BLE001
+                status = _http_status_of(err)
+                # 404: nothing to void; 409: already settled; 4xx auth: permanent.
+                if status is not None and 400 <= status < 500:
+                    return False
+                if attempt + 1 < self._POST_ATTEMPTS:
+                    await asyncio.sleep(0.25 * (attempt + 1))
+        # Outcome unknown: keep the reservation and retry before the next push / at exit.
+        self._pending_voids.append((event_name, operation_id, reserved))
+        logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
+        return False
+
+    async def _retry_pending_voids(self) -> bool:
+        """Retry unconfirmed voids (caller holds the operation lock).
+
+        Returns False while any earlier void is still unconfirmed.
+        """
+        pending, self._pending_voids = self._pending_voids, []
+        for event_name, operation_id, reserved in pending:
+            await self._void_charge(event_name, operation_id, reserved)
+        return not self._pending_voids
+
     def _quarantine_stale_pending(self) -> None:
         """Remove intents older than the safe replay window (fail closed, no replay).
 
@@ -544,6 +593,7 @@ class ChargingManager:
         """Caller must hold _operation_lock."""
         if not self._is_at_home or self._client is None or not self._actor_run_id:
             return 0
+        await self._retry_pending_voids()
 
         import time
 
@@ -731,6 +781,8 @@ class ChargingManager:
                 )
             if accepted is not None:
                 self._drop_pending(idempotency_key)
+                if accepted > 0:
+                    self._last_accepted_operation_id = idempotency_key
                 if accepted < chargeable_count:
                     self._unwind_reservation(event_name, chargeable_count - accepted)
                     chargeable_count = accepted
@@ -1032,6 +1084,9 @@ async def push_data_and_charge(
     async with charging_manager._operation_lock:
         if charging_manager._closed:
             raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
+        # A void that did not go through for an earlier failed push must land
+        # before this push charges again, or one unsaved result is billed twice.
+        voids_confirmed = await charging_manager._retry_pending_voids()
         # Non-PPE actors push everything without charging (JS calculatePushDataLimits
         # parity: `if (!this.isPayPerEvent) return { limitedItems, eventsToCharge: {} }`).
         is_ppe = charging_manager._pricing_info is not None
@@ -1065,6 +1120,29 @@ async def push_data_and_charge(
                 event_charge_limit_reached=len(items) > 0 and limited_count == 0,
             )
 
+        # Every platform event this push would charge counts: the explicit event
+        # and, on a default-dataset push, the configured default event. The
+        # synthetic default-dataset event (local bookkeeping) and unpriced
+        # events (never posted) do not.
+        charged_names: List[str] = []
+        if event_name:
+            charged_names.append(event_name)
+        if is_default_dataset:
+            configured_default = charging_manager._config.get('default_dataset_item_event_name')
+            if configured_default:
+                charged_names.append(configured_default)
+        if (
+            not voids_confirmed
+            and is_ppe
+            and any(
+                not is_default_dataset_item_event(name) and name in charging_manager._per_event_prices
+                for name in charged_names
+            )
+        ):
+            raise RuntimeError(
+                'An earlier charge for a failed dataset push could not be voided yet; retry the push'
+            )
+
         # Charge BEFORE persisting so a server cap/reject cannot leave unbilled
         # items in the dataset. Operation lock is held across both steps.
         if is_ppe and resolved_event:
@@ -1079,15 +1157,20 @@ async def push_data_and_charge(
 
             # Synthetic storage event first (local-only); then explicit billable event.
             reserved_in_op: list[tuple[str, int]] = []
+            operation_ids: list[Optional[str]] = []
             try:
                 if default_event:
+                    charging_manager._take_last_accepted_operation_id()
                     r = await charging_manager._charge_exclusive(default_event, len(limited_items))
                     results.append(r)
                     reserved_in_op.append((default_event, r.charged_count))
+                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
                 if event_name and event_name != default_event:
+                    charging_manager._take_last_accepted_operation_id()
                     r = await charging_manager._charge_exclusive(event_name, len(limited_items))
                     results.append(r)
                     reserved_in_op.append((event_name, r.charged_count))
+                    operation_ids.append(charging_manager._take_last_accepted_operation_id())
             except Exception:
                 for name, reserved in reserved_in_op:
                     if reserved > 0:
@@ -1118,11 +1201,27 @@ async def push_data_and_charge(
                     try:
                         await push_fn(limited_items[:accepted_for_items])
                     except Exception as push_err:
+                        # A result is billed only if it was saved: void this
+                        # operation's platform charges and unwind local
+                        # reservations, then let the caller retry the push.
                         logger.error(
-                            'Dataset push failed after charge accepted (items=%s): %s',
+                            'Dataset push failed after charge accepted (items=%s): %s; voiding the charge',
                             accepted_for_items,
                             push_err,
                         )
+                        # Only when the push certainly stored nothing (the storage API
+                        # answered with an error, or the request was never sent). A
+                        # timeout after sending may have stored the items.
+                        if not _push_certainly_failed(push_err):
+                            raise
+                        for (name, _), op_id in zip(reserved_in_op, operation_ids):
+                            kept = min(accepted_for_items, next(r.charged_count for n, r in charged_events if n == name))
+                            if kept <= 0:
+                                continue
+                            if op_id:
+                                await charging_manager._void_charge(name, op_id, kept)
+                            elif is_default_dataset_item_event(name):
+                                charging_manager._unwind_reservation(name, kept)
                         raise
                 return ChargeResult(
                     chargeable_within_limit=charging_manager.compute_chargeable(),
@@ -1141,3 +1240,35 @@ async def push_data_and_charge(
             charged_count=0,
             event_charge_limit_reached=False,
         )
+
+
+def _http_status_of(err: BaseException) -> Optional[int]:
+    status = getattr(err, 'status_code', None)
+    if status is None:
+        response = getattr(err, 'response', None)
+        status = getattr(response, 'status_code', None) if response is not None else None
+    try:
+        return int(status) if status is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _push_certainly_failed(err: BaseException) -> bool:
+    """True when a dataset push certainly stored nothing.
+
+    The storage API answered with an error status, or the connection was never
+    made. Timeouts and resets after sending are not proof: the write may have landed.
+    """
+    import httpx
+
+    e: Optional[BaseException] = err
+    for _ in range(4):
+        if e is None:
+            break
+        status = _http_status_of(e)
+        if status is not None:
+            return status >= 400
+        if isinstance(e, (httpx.ConnectError, ConnectionRefusedError)):
+            return True
+        e = e.__cause__ or e.__context__
+    return False

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely/proxy_configuration.py

@@ -108,6 +108,25 @@ def _sanitize_fragment(value: str) -> str:
     return re.sub(r'[:@/?#\s]', '', value)
 
 
+def _input_use_platform_proxy(actor_proxy_input: Dict[str, Any]) -> Optional[bool]:
+    """The proxy input field's platform-proxy switch, or None when it has none.
+
+    `useScrapelyProxy` wins when it is a boolean; otherwise the legacy
+    `useApifyProxy` key of existing inputs and ported actors decides.
+    """
+    for key in ('useScrapelyProxy', 'useApifyProxy'):
+        value = actor_proxy_input.get(key)
+        if isinstance(value, bool):
+            return value
+    return None
+
+
+def _input_proxy_option(actor_proxy_input: Dict[str, Any], name: str) -> Any:
+    """`scrapelyProxy<name>` of the proxy input field when it is set, else the legacy `apifyProxy<name>`."""
+    value = actor_proxy_input.get(f'scrapelyProxy{name}')
+    return value if value is not None else actor_proxy_input.get(f'apifyProxy{name}')
+
+
 def _platform_env() -> Optional[Dict[str, Any]]:
     """Read platform proxy credentials from the environment (or None)."""
     hostname = os.environ.get('SCRAPELY_PROXY_HOSTNAME') or os.environ.get('APIFY_PROXY_HOSTNAME')
@@ -213,10 +232,10 @@ class ProxyConfiguration:
 
         # If actor_proxy_input is provided, extract settings from it
         if actor_proxy_input:
-            if actor_proxy_input.get('useApifyProxy', False):
-                self._groups = actor_proxy_input.get('apifyProxyGroups', []) or []
-                self._country_code = actor_proxy_input.get('apifyProxyCountry')
-                self._subdivision_code = actor_proxy_input.get('apifyProxySubdivision')
+            if _input_use_platform_proxy(actor_proxy_input):
+                self._groups = _input_proxy_option(actor_proxy_input, 'Groups') or []
+                self._country_code = _input_proxy_option(actor_proxy_input, 'Country')
+                self._subdivision_code = _input_proxy_option(actor_proxy_input, 'Subdivision')
             else:
                 self._proxy_urls = actor_proxy_input.get('proxyUrls', []) or []
                 # Tiered lists may also arrive through actor input (future-proof
@@ -291,7 +310,7 @@ class ProxyConfiguration:
         )
         if has_real_custom and (self._groups or self._country_code or self._subdivision_code or password):
             raise ValueError(
-                'Cannot combine custom proxies with Apify Proxy! It is not allowed to set '
+                'Cannot combine custom proxies with Scrapely Proxy! It is not allowed to set '
                 '"proxy_urls" or "new_url_function" combined with "groups", "country_code" '
                 'or "subdivision_code".'
             )
@@ -480,7 +499,8 @@ class ProxyConfiguration:
         Sends ``GET {status_url}/?format=json`` *through the platform gateway``
         so the proxy handshake itself authenticates the run's credentials.
         Mirrors the JS SDK: 4s timeout, 2 attempts, returns ``None`` when the
-        endpoint stays unreachable. Never raises.
+        endpoint stays unreachable. Never raises. ``fromPayload`` is True when
+        the gateway answered with its own status JSON rather than an HTTP error.
         """
         import urllib.error
         import urllib.request
@@ -514,6 +534,7 @@ class ProxyConfiguration:
                     'connected': payload.get('connected') is True,
                     'connectionError': payload.get('connectionError') or '',
                     'isManInTheMiddle': payload.get('isManInTheMiddle') is True,
+                    'fromPayload': True,
                 }
             except urllib.error.HTTPError as e:
                 if e.code == 407:
@@ -524,6 +545,7 @@ class ProxyConfiguration:
                             'SCRAPELY_PROXY_*/APIFY_PROXY_* credentials.'
                         ),
                         'isManInTheMiddle': False,
+                        'fromPayload': False,
                     }
                 if 590 <= e.code <= 599:
                     description = PROXY_STATUS_CODE_DESCRIPTIONS.get(e.code, 'upstream error')
@@ -531,6 +553,7 @@ class ProxyConfiguration:
                         'connected': False,
                         'connectionError': f'Platform proxy upstream error (HTTP {e.code}: {description}).',
                         'isManInTheMiddle': False,
+                        'fromPayload': False,
                     }
                 # Other HTTP errors behave like connection errors: retry.
             except Exception:  # noqa: BLE001 - retry connection errors (Apify behavior)
@@ -623,6 +646,9 @@ class ProxyConfiguration:
 
         Raises:
             ValueError: If proxy configuration is invalid
+            ConnectionError: If the platform proxy reports that this configuration
+                cannot be used (for example an unknown proxy group), with the
+                proxy's ``connectionError`` as the message (Apify parity)
         """
         if self._initialized:
             return
@@ -639,9 +665,10 @@ class ProxyConfiguration:
         if self._proxy_urls and self._new_url_function:
             raise ValueError('Cannot specify both proxy_urls and new_url_function')
 
-        # Apify-parity access check (best-effort, never fatal): GET
-        # {status_url}/?format=json *through the platform gateway* so the proxy
-        # handshake itself authenticates the run's credentials. Parity with the
+        # Apify-parity access check: GET {status_url}/?format=json *through the
+        # platform gateway* so the proxy handshake itself authenticates the run's
+        # credentials. A "connected": false answer raises like Apify's
+        # _check_access; timeouts and HTTP errors only warn. Parity with the
         # Apify SDKs (usesApifyProxy / uses_apify_proxy) and the JS Scrapely
         # port (platformEnv): the check only applies when this configuration
         # actually uses the platform proxy — BYOP-only configurations skip it
@@ -662,6 +689,8 @@ class ProxyConfiguration:
                 )
             else:
                 self._is_man_in_the_middle = status['isManInTheMiddle']
+                if not status['connected'] and status['fromPayload']:
+                    raise ConnectionError(status['connectionError'])
                 if not status['connected']:
                     logger.warning('Platform proxy access check failed: %s', status['connectionError'])
                 else:

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely_python.egg-info/PKG-INFO

@@ -1,16 +1,15 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.1
+Version: 0.1.3
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
-License: MIT
+License-Expression: MIT
 Project-URL: Homepage, https://scrape.ly
 Project-URL: Documentation, https://docs.scrape.ly
 Project-URL: Repository, https://github.com/SandeepDev1/scrapely
 Keywords: scraping,actor,serverless,apify,scrapely
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
@@ -22,7 +21,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: httpx>=0.25.0
 Requires-Dist: crawlee<2.0.0,>=1.8.0
 Requires-Dist: websockets>=12.0
-Requires-Dist: scrapely-client-python<2,>=1.0.0
+Requires-Dist: scrapely-client-python<2,>=1.0.2
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: cryptography>=42.0.0
 Provides-Extra: dev

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/src/scrapely_python.egg-info/requires.txt

@@ -1,7 +1,7 @@
 httpx>=0.25.0
 crawlee<2.0.0,>=1.8.0
 websockets>=12.0
-scrapely-client-python<2,>=1.0.0
+scrapely-client-python<2,>=1.0.2
 pydantic>=2.0.0
 cryptography>=42.0.0
 

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/tests/test_charging_pending.py

@@ -457,3 +457,205 @@ async def test_dual_event_push_unwinds_synthetic_on_partial_explicit() -> None:
     assert len(pushed) == 1
     assert mgr.get_charged_event_count('page-open') == 1
     assert mgr.get_charged_event_count('apify-default-dataset-item') == 1
+
+
+@pytest.mark.asyncio
+async def test_push_failure_after_charge_voids_the_charge() -> None:
+    """A result is billed only if it was saved: a failed push voids its charge."""
+    client = FakeClient()
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 503
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('503 from dataset push')
+
+    with pytest.raises(PushHttpError, match='503'):
+        await push_data_and_charge(mgr, [{'a': 1}, {'a': 2}], 'page-open', False, failing_push)
+
+    charge_posts = [p for p in client.posts if p['path'].endswith('/charge')]
+    void_posts = [p for p in client.posts if p['path'].endswith('/charge/void')]
+    assert len(charge_posts) == 1
+    assert len(void_posts) == 1
+    assert void_posts[0]['path'] == '/v2/actor-runs/run-test-1/charge/void'
+    assert void_posts[0]['headers']['Idempotency-Key'] == charge_posts[0]['headers']['Idempotency-Key']
+    assert mgr.get_charged_event_count('page-open') == 0
+
+    pushed: List[Any] = []
+
+    async def ok_push(items: Any) -> None:
+        pushed.extend(items)
+
+    await push_data_and_charge(mgr, [{'a': 3}], 'page-open', False, ok_push)
+    assert len([p for p in client.posts if p['path'].endswith('/charge/void')]) == 1
+    assert mgr.get_charged_event_count('page-open') == 1
+    assert pushed == [{'a': 3}]
+
+
+
+class VoidFlakyClient(FakeClient):
+    def __init__(self, void_failures: int = 0) -> None:
+        super().__init__()
+        self.void_failures = void_failures
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **kwargs: Any) -> Dict[str, Any]:
+        if path.endswith('/charge/void') and self.void_failures > 0:
+            self.void_failures -= 1
+            self.posts.append({'path': path + '#failed', 'json': json_data, 'headers': headers or {}})
+            raise TimeoutError('void timed out')
+        return await super().post(path, json_data=json_data, headers=headers, **kwargs)
+
+
+@pytest.mark.asyncio
+async def test_push_timeout_after_send_keeps_the_charge() -> None:
+    client = FakeClient()
+    mgr = _make_manager(client)
+
+    async def reset_push(_items: Any) -> None:
+        raise TimeoutError('read timed out')  # may have been stored
+
+    with pytest.raises(TimeoutError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, reset_push)
+    assert not [p for p in client.posts if p['path'].endswith('/charge/void')]
+    assert mgr.get_charged_event_count('page-open') == 1
+
+
+@pytest.mark.asyncio
+async def test_unconfirmed_void_is_retried_before_next_charge() -> None:
+    client = VoidFlakyClient(void_failures=1)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+    assert mgr.get_charged_event_count('page-open') == 1  # void unconfirmed: reservation kept
+
+    async def ok_push(_items: Any) -> None:
+        return None
+
+    await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, ok_push)
+    paths = [p['path'].rsplit('/', 1)[-1] for p in client.posts]
+    assert paths == ['charge', 'void#failed', 'void', 'charge']
+    assert mgr.get_charged_event_count('page-open') == 1
+
+
+
+@pytest.mark.asyncio
+async def test_new_push_refused_while_earlier_void_unconfirmed() -> None:
+    client = VoidFlakyClient(void_failures=99)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+
+    async def ok_push(_items: Any) -> None:
+        return None
+
+    with pytest.raises(RuntimeError, match='could not be voided yet'):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, ok_push)
+    assert len([p for p in client.posts if p['path'].endswith('/charge')]) == 1
+
+
+@pytest.mark.asyncio
+async def test_uncharged_push_not_blocked_by_unconfirmed_void() -> None:
+    client = VoidFlakyClient(void_failures=99)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+
+    pushed: List[Any] = []
+
+    async def ok_push(items: Any) -> None:
+        pushed.append(items)
+
+    await push_data_and_charge(mgr, [{'b': 1}], None, False, ok_push)
+    assert len(pushed) == 1
+
+
+@pytest.mark.asyncio
+async def test_configured_platform_default_event_is_gated_by_unconfirmed_void() -> None:
+    client = VoidFlakyClient(void_failures=99)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+
+    # A configured default event that is a real platform event must be gated too.
+    mgr._config.get = MagicMock(side_effect=lambda key, default=None: 'page-open' if key == 'default_dataset_item_event_name' else None)
+
+    async def ok_push(_items: Any) -> None:
+        return None
+
+    with pytest.raises(RuntimeError, match='could not be voided yet'):
+        await push_data_and_charge(mgr, [{'b': 1}], None, True, ok_push)
+
+
+@pytest.mark.asyncio
+async def test_unpriced_configured_default_event_not_blocked() -> None:
+    client = VoidFlakyClient(void_failures=99)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+    mgr._config.get = MagicMock(side_effect=lambda key, default=None: 'not-priced' if key == 'default_dataset_item_event_name' else None)
+    pushed: List[Any] = []
+
+    async def ok_push(items: Any) -> None:
+        pushed.append(items)
+
+    await push_data_and_charge(mgr, [{'b': 1}], None, True, ok_push)
+    assert len(pushed) == 1
+
+
+@pytest.mark.asyncio
+async def test_unpriced_explicit_event_with_priced_default_is_gated() -> None:
+    client = VoidFlakyClient(void_failures=99)
+    mgr = _make_manager(client)
+
+    class PushHttpError(Exception):
+        status_code = 502
+
+    async def failing_push(_items: Any) -> None:
+        raise PushHttpError('bad gateway')
+
+    with pytest.raises(PushHttpError):
+        await push_data_and_charge(mgr, [{'a': 1}], 'page-open', False, failing_push)
+    # Configured default event is priced; the explicit event is not.
+    mgr._config.get = MagicMock(side_effect=lambda key, default=None: 'page-open' if key == 'default_dataset_item_event_name' else None)
+
+    async def ok_push(_items: Any) -> None:
+        return None
+
+    with pytest.raises(RuntimeError, match='could not be voided yet'):
+        await push_data_and_charge(mgr, [{'b': 1}], 'not-priced', True, ok_push)

{scrapely_python-0.1.1 → scrapely_python-0.1.3}/tests/test_proxy_configuration.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import json
+
 import pytest
 
 from scrapely.proxy_configuration import ProxyConfiguration
@@ -87,7 +89,7 @@ async def test_sentinel_without_env_raises(monkeypatch: pytest.MonkeyPatch) -> N
 
 
 async def test_real_custom_plus_groups_rejected() -> None:
-    with pytest.raises(ValueError, match='Cannot combine custom proxies with Apify Proxy'):
+    with pytest.raises(ValueError, match='Cannot combine custom proxies with Scrapely Proxy'):
         ProxyConfiguration(proxy_urls=['http://user:pass@byok.example.com:8080'], groups=['RESIDENTIAL'])
     # Sentinel-only list may combine with groups (platform slot options).
     pc = ProxyConfiguration(proxy_urls=['auto'], groups=['RESIDENTIAL'])
@@ -222,7 +224,7 @@ async def test_subdivision_and_country_validation() -> None:
     with pytest.raises(ValueError, match='1-3 character ISO 3166-2'):
         ProxyConfiguration(country_code='US', subdivision_code='ca')
     # Subdivision combined with real BYOP URLs is a conflicting combination.
-    with pytest.raises(ValueError, match='Cannot combine custom proxies with Apify Proxy'):
+    with pytest.raises(ValueError, match='Cannot combine custom proxies with Scrapely Proxy'):
         ProxyConfiguration(
             proxy_urls=['http://byok.example.com:8080'],
             country_code='US',
@@ -500,6 +502,50 @@ async def test_access_check_runs_for_mixed_byop_and_sentinel(
     assert any('access check succeeded' in r.message for r in caplog.records)
 
 
+UNKNOWN_GROUP_ERROR = (
+    'Proxy group "GOOGLE_SERP" is not available on Scrapely. Available groups: RESIDENTIAL, DATACENTER.'
+)
+REFUSED_PAYLOAD = json.dumps({'connected': False, 'connectionError': UNKNOWN_GROUP_ERROR, 'isManInTheMiddle': False})
+CONNECTED_PAYLOAD = '{"connected": true, "isManInTheMiddle": false}'
+
+
+def _proxy_username(auth: str | None) -> str:
+    import base64
+
+    return base64.b64decode((auth or '').removeprefix('Basic ')).decode().split(':')[0]
+
+
+async def test_access_check_raises_gateway_connection_error(monkeypatch: pytest.MonkeyPatch) -> None:
+    with FakeGateway(200, REFUSED_PAYLOAD) as gateway:
+        _point_env_at_gateway(monkeypatch, gateway)
+        pc = ProxyConfiguration(groups=['GOOGLE_SERP'])
+        with pytest.raises(ConnectionError) as excinfo:
+            await pc.initialize()
+
+    assert str(excinfo.value) == UNKNOWN_GROUP_ERROR
+    assert [_proxy_username(auth) for _, auth in gateway.requests] == ['sub1,groups-GOOGLE_SERP']
+
+
+async def test_access_check_known_groups_succeed(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    import logging
+
+    with FakeGateway(200, CONNECTED_PAYLOAD) as gateway:
+        _point_env_at_gateway(monkeypatch, gateway)
+        with caplog.at_level(logging.INFO, logger='scrapely.proxy_configuration'):
+            for groups in (None, ['RESIDENTIAL'], ['DATACENTER']):
+                await ProxyConfiguration(groups=groups).initialize()
+
+    assert [_proxy_username(auth) for _, auth in gateway.requests] == [
+        'sub1',
+        'sub1,groups-RESIDENTIAL',
+        'sub1,groups-DATACENTER',
+    ]
+    assert sum('access check succeeded' in r.message for r in caplog.records) == 3
+    assert not any(r.levelno >= logging.WARNING for r in caplog.records)
+
+
 
 # ── Actor.create_proxy_configuration useApifyProxy semantics (Node parity) ────
 
@@ -567,6 +613,98 @@ async def test_actor_create_use_apify_proxy_false_with_explicit_kwarg_keeps_byop
     assert await pc.new_url() == 'http://byop-b:8080'
 
 
+# useScrapelyProxy is the canonical key; a boolean useScrapelyProxy wins over useApifyProxy.
+
+async def test_actor_create_use_scrapely_proxy_true_uses_platform(initialized_actor) -> None:
+    pc = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': True, 'apifyProxyGroups': ['RESIDENTIAL']},
+        check_access=False,
+    )
+    assert pc is not None
+    assert 'groups-RESIDENTIAL' in await pc.new_url()
+
+
+async def test_actor_create_use_scrapely_proxy_false_disables(initialized_actor) -> None:
+    pc = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': False, 'apifyProxyGroups': ['RESIDENTIAL']},
+        check_access=False,
+    )
+    assert pc is None
+
+
+async def test_actor_create_use_scrapely_proxy_false_with_proxy_urls_keeps_byop(initialized_actor) -> None:
+    pc = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': False, 'proxyUrls': ['http://byop-a:8080']},
+        check_access=False,
+    )
+    assert pc is not None
+    assert await pc.new_url() == 'http://byop-a:8080'
+
+
+async def test_actor_create_use_scrapely_proxy_wins_over_use_apify_proxy(initialized_actor) -> None:
+    off = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': False, 'useApifyProxy': True},
+        check_access=False,
+    )
+    assert off is None
+    on = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': True, 'useApifyProxy': False, 'apifyProxyGroups': ['RESIDENTIAL']},
+        check_access=False,
+    )
+    assert on is not None
+    assert 'groups-RESIDENTIAL' in await on.new_url()
+
+
+def test_proxy_configuration_actor_input_use_scrapely_proxy() -> None:
+    pc = ProxyConfiguration(actor_proxy_input={'useScrapelyProxy': True, 'apifyProxyGroups': ['RESIDENTIAL']})
+    assert pc._groups == ['RESIDENTIAL']
+    custom = ProxyConfiguration(actor_proxy_input={'useScrapelyProxy': False, 'proxyUrls': ['http://byop-c:8080']})
+    assert custom._proxy_urls == ['http://byop-c:8080']
+    assert custom._groups == []
+
+
+def test_proxy_configuration_actor_input_scrapely_proxy_options() -> None:
+    pc = ProxyConfiguration(actor_proxy_input={
+        'useScrapelyProxy': True,
+        'scrapelyProxyGroups': ['RESIDENTIAL'],
+        'scrapelyProxyCountry': 'US',
+        'scrapelyProxySubdivision': 'CA',
+    })
+    assert pc._groups == ['RESIDENTIAL']
+    assert pc._country_code == 'US'
+    assert pc._subdivision_code == 'CA'
+    both = ProxyConfiguration(actor_proxy_input={
+        'useScrapelyProxy': True,
+        'scrapelyProxyGroups': ['RESIDENTIAL'],
+        'apifyProxyGroups': ['DATACENTER'],
+        'apifyProxyCountry': 'DE',
+    })
+    assert both._groups == ['RESIDENTIAL']
+    assert both._country_code == 'DE'
+
+
+async def test_actor_create_scrapely_proxy_groups_uses_platform(initialized_actor) -> None:
+    pc = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useScrapelyProxy': True, 'scrapelyProxyGroups': ['RESIDENTIAL'], 'scrapelyProxyCountry': 'US'},
+        check_access=False,
+    )
+    assert pc is not None
+    url = await pc.new_url()
+    assert 'groups-RESIDENTIAL' in url
+    assert 'country-US' in url
+
+
+async def test_actor_create_legacy_apify_proxy_groups_still_work(initialized_actor) -> None:
+    pc = await initialized_actor.create_proxy_configuration(
+        actor_proxy_input={'useApifyProxy': True, 'apifyProxyGroups': ['RESIDENTIAL'], 'apifyProxyCountry': 'US'},
+        check_access=False,
+    )
+    assert pc is not None
+    url = await pc.new_url()
+    assert 'groups-RESIDENTIAL' in url
+    assert 'country-US' in url
+
+
 async def test_actor_create_no_options_without_env_returns_none(
     initialized_actor, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -577,6 +715,37 @@ async def test_actor_create_no_options_without_env_returns_none(
     assert pc is None
 
 
+async def test_actor_create_refused_group_raises(initialized_actor, monkeypatch: pytest.MonkeyPatch) -> None:
+    with FakeGateway(200, REFUSED_PAYLOAD) as gateway:
+        _point_env_at_gateway(monkeypatch, gateway)
+        with pytest.raises(ConnectionError, match='Proxy group "GOOGLE_SERP" is not available'):
+            await initialized_actor.create_proxy_configuration(groups=['GOOGLE_SERP'])
+        with pytest.raises(ConnectionError, match='Proxy group "GOOGLE_SERP" is not available'):
+            await initialized_actor.create_proxy_configuration(
+                actor_proxy_input={'useApifyProxy': True, 'apifyProxyGroups': ['GOOGLE_SERP']}
+            )
+
+    assert [_proxy_username(auth) for _, auth in gateway.requests] == [
+        'sub1,groups-GOOGLE_SERP',
+        'sub1,groups-GOOGLE_SERP',
+    ]
+
+
+async def test_actor_create_http_errors_and_timeouts_only_warn(
+    initialized_actor, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    import logging
+
+    with caplog.at_level(logging.WARNING, logger='scrapely.proxy_configuration'):
+        with FakeGateway(407, '') as gateway:
+            _point_env_at_gateway(monkeypatch, gateway)
+            assert await initialized_actor.create_proxy_configuration(groups=['GOOGLE_SERP']) is not None
+        assert await initialized_actor.create_proxy_configuration(groups=['GOOGLE_SERP']) is not None
+
+    assert any('authentication failed (HTTP 407)' in r.message for r in caplog.records)
+    assert any('access check timed out' in r.message for r in caplog.records)
+
+
 async def test_actor_create_proxy_urls_from_input_without_flag(initialized_actor) -> None:
     pc = await initialized_actor.create_proxy_configuration(
         actor_proxy_input={'proxyUrls': ['http://byop-a:8080']},