scrapely-python 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.2'
+__version__ = '0.1.4'
 __all__ = [
     # Core
     'Actor',

scrapely/actor.py

@@ -1776,7 +1776,7 @@ class Actor(metaclass=_ActorMeta):
         """
         cls._ensure_initialized('create_proxy_configuration')
 
-        from .proxy_configuration import ProxyConfiguration, _input_use_platform_proxy
+        from .proxy_configuration import ProxyConfiguration, _input_proxy_option, _input_use_platform_proxy
 
         # Handle actor_proxy_input (Node SDK normalizeProxyInput/createProxyConfiguration
         # parity): `useScrapelyProxy: false` (or the legacy `useApifyProxy: false`; a
@@ -1787,7 +1787,7 @@ class Actor(metaclass=_ActorMeta):
         # silently drop the user's own proxies and crawl on their real IPs. We only
         # bail when no custom proxies are given either. A MISSING key falls back to
         # platform mode when the server injected credentials — input like
-        # {apifyProxyGroups: [...]} must not silently disable proxying.
+        # {scrapelyProxyGroups: [...]} must not silently disable proxying.
         if actor_proxy_input is not None:
             if _input_use_platform_proxy(actor_proxy_input) is False and not (
                 proxy_urls
@@ -1799,9 +1799,9 @@ class Actor(metaclass=_ActorMeta):
                 or actor_proxy_input.get('tieredProxyConfig')
             ):
                 return None
-            country_code = country_code or actor_proxy_input.get('apifyProxyCountry')
-            subdivision_code = subdivision_code or actor_proxy_input.get('apifyProxySubdivision')
-            groups = groups or actor_proxy_input.get('apifyProxyGroups')
+            country_code = country_code or _input_proxy_option(actor_proxy_input, 'Country')
+            subdivision_code = subdivision_code or _input_proxy_option(actor_proxy_input, 'Subdivision')
+            groups = groups or _input_proxy_option(actor_proxy_input, 'Groups')
             proxy_urls = proxy_urls or actor_proxy_input.get('proxyUrls')
             tiered_proxy_urls = tiered_proxy_urls or actor_proxy_input.get('tieredProxyUrls')
             tiered_proxy_config = tiered_proxy_config or actor_proxy_input.get('tieredProxyConfig')

scrapely/charging.py

@@ -195,6 +195,11 @@ class ChargingManager:
         self._pending_charge_keys: Dict[str, Dict[str, Any]] = {}
         # Optional local charging-log dataset (ACTOR_USE_CHARGING_LOG_DATASET).
         self._charging_log_dataset: Any = None
+        # Idempotency key of the last charge the platform accepted
+        # (push_data_and_charge voids it when the push then fails).
+        self._last_accepted_operation_id: Optional[str] = None
+        # Voids not yet confirmed by the platform; retried before the next push and at exit.
+        self._pending_voids: List[tuple[str, str, int, bool]] = []
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
@@ -429,6 +434,67 @@ class ChargingManager:
                 self._total_charged_usd - event_price * Decimal(count),
             )
 
+    def _take_last_accepted_operation_id(self) -> Optional[str]:
+        op_id = self._last_accepted_operation_id
+        self._last_accepted_operation_id = None
+        return op_id
+
+    async def _cancel_uncertain_charge(self, event_name: str, operation_id: str, reserved: int) -> bool:
+        """Cancel an unconfirmed charge for items that were then not stored.
+
+        Drops the pending intent (so the exit flush never posts it) and voids it in
+        case the original request did apply; a 404 means it never applied.
+        """
+        self._drop_pending(operation_id)
+        return await self._void_charge(event_name, operation_id, reserved, not_found_means_voided=True)
+
+    async def _void_charge(
+        self, event_name: str, operation_id: str, reserved: int, not_found_means_voided: bool = False
+    ) -> bool:
+        """Reverse an accepted charge whose items were never stored, then unwind locally.
+
+        POST /v2/actor-runs/{run}/charge/void with the charge's Idempotency-Key.
+        The local reservation is released only once the platform confirms.
+        """
+        if self._client is None or not self._is_at_home or not self._actor_run_id:
+            return False
+        for attempt in range(self._POST_ATTEMPTS):
+            try:
+                await self._client.post(
+                    f'/v2/actor-runs/{self._actor_run_id}/charge/void',
+                    json_data={},
+                    headers={'Idempotency-Key': operation_id},
+                )
+                self._unwind_reservation(event_name, reserved)
+                return True
+            except Exception as err:  # noqa: BLE001
+                status = _http_status_of(err)
+                # Only the void route's own answer proves it never applied; a server
+                # without the route (older deployment) also answers 404.
+                if status == 404 and not_found_means_voided and getattr(err, 'type', None) == 'charge-not-found':
+                    # The unconfirmed charge never reached the ledger: nothing billed.
+                    self._unwind_reservation(event_name, reserved)
+                    return True
+                # 404: nothing to void; 409: already settled; 4xx auth: permanent.
+                if status is not None and 400 <= status < 500:
+                    return False
+                if attempt + 1 < self._POST_ATTEMPTS:
+                    await asyncio.sleep(0.25 * (attempt + 1))
+        # Outcome unknown: keep the reservation and retry before the next push / at exit.
+        self._pending_voids.append((event_name, operation_id, reserved, not_found_means_voided))
+        logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
+        return False
+
+    async def _retry_pending_voids(self) -> bool:
+        """Retry unconfirmed voids (caller holds the operation lock).
+
+        Returns False while any earlier void is still unconfirmed.
+        """
+        pending, self._pending_voids = self._pending_voids, []
+        for event_name, operation_id, reserved, not_found_means_voided in pending:
+            await self._void_charge(event_name, operation_id, reserved, not_found_means_voided)
+        return not self._pending_voids
+
     def _quarantine_stale_pending(self) -> None:
         """Remove intents older than the safe replay window (fail closed, no replay).
 
@@ -544,6 +610,7 @@ class ChargingManager:
         """Caller must hold _operation_lock."""
         if not self._is_at_home or self._client is None or not self._actor_run_id:
             return 0
+        await self._retry_pending_voids()
 
         import time
 
@@ -731,6 +798,8 @@ class ChargingManager:
                 )
             if accepted is not None:
                 self._drop_pending(idempotency_key)
+                if accepted > 0:
+                    self._last_accepted_operation_id = idempotency_key
                 if accepted < chargeable_count:
                     self._unwind_reservation(event_name, chargeable_count - accepted)
                     chargeable_count = accepted
@@ -1032,6 +1101,9 @@ async def push_data_and_charge(
     async with charging_manager._operation_lock:
         if charging_manager._closed:
             raise RuntimeError('ChargingManager is closed (actor is shutting down); charge rejected')
+        # A void that did not go through for an earlier failed push must land
+        # before this push charges again, or one unsaved result is billed twice.
+        voids_confirmed = await charging_manager._retry_pending_voids()
         # Non-PPE actors push everything without charging (JS calculatePushDataLimits
         # parity: `if (!this.isPayPerEvent) return { limitedItems, eventsToCharge: {} }`).
         is_ppe = charging_manager._pricing_info is not None
@@ -1065,6 +1137,29 @@ async def push_data_and_charge(
                 event_charge_limit_reached=len(items) > 0 and limited_count == 0,
             )
 
+        # Every platform event this push would charge counts: the explicit event
+        # and, on a default-dataset push, the configured default event. The
+        # synthetic default-dataset event (local bookkeeping) and unpriced
+        # events (never posted) do not.
+        charged_names: List[str] = []
+        if event_name:
+            charged_names.append(event_name)
+        if is_default_dataset:
+            configured_default = charging_manager._config.get('default_dataset_item_event_name')
+            if configured_default:
+                charged_names.append(configured_default)
+        if (
+            not voids_confirmed
+            and is_ppe
+            and any(
+                not is_default_dataset_item_event(name) and name in charging_manager._per_event_prices
+                for name in charged_names
+            )
+        ):
+            raise RuntimeError(
+                'An earlier charge for a failed dataset push could not be voided yet; retry the push'
+            )
+
         # Charge BEFORE persisting so a server cap/reject cannot leave unbilled
         # items in the dataset. Operation lock is held across both steps.
         if is_ppe and resolved_event:
@@ -1079,15 +1174,40 @@ async def push_data_and_charge(
 
             # Synthetic storage event first (local-only); then explicit billable event.
             reserved_in_op: list[tuple[str, int]] = []
+            operation_ids: list[Optional[str]] = []
+            uncertain_ops: set[str] = set()
+
+            async def charge_one(name: str) -> None:
+                charging_manager._take_last_accepted_operation_id()
+                try:
+                    r = await charging_manager._charge_exclusive(name, len(limited_items))
+                    op_id = charging_manager._take_last_accepted_operation_id()
+                except ChargeUncertainError as unconfirmed:
+                    # The platform did not confirm the charge (5xx / timeouts after
+                    # retries). Never drop the items for that: the intent stays
+                    # pending with its idempotency key and is re-posted by the exit
+                    # flush, so the items are stored now and billed once.
+                    logger.warning(
+                        'Charge for %r not confirmed yet (%s); storing the items and retrying the charge later',
+                        name,
+                        unconfirmed.operation_id,
+                    )
+                    r = ChargeResult(
+                        chargeable_within_limit=charging_manager.compute_chargeable(),
+                        charged_count=unconfirmed.count,
+                        event_charge_limit_reached=False,
+                    )
+                    op_id = unconfirmed.operation_id
+                    uncertain_ops.add(op_id)
+                results.append(r)
+                reserved_in_op.append((name, r.charged_count))
+                operation_ids.append(op_id)
+
             try:
                 if default_event:
-                    r = await charging_manager._charge_exclusive(default_event, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((default_event, r.charged_count))
+                    await charge_one(default_event)
                 if event_name and event_name != default_event:
-                    r = await charging_manager._charge_exclusive(event_name, len(limited_items))
-                    results.append(r)
-                    reserved_in_op.append((event_name, r.charged_count))
+                    await charge_one(event_name)
             except Exception:
                 for name, reserved in reserved_in_op:
                     if reserved > 0:
@@ -1109,20 +1229,59 @@ async def push_data_and_charge(
                 )
                 # Unwind any event that reserved more than we will persist
                 # (synthetic default-dataset is local-only and otherwise sticks).
+                platform_op_names = {
+                    name for (name, _), op_id in zip(reserved_in_op, operation_ids) if op_id
+                }
                 for name, result in charged_events:
                     if result.charged_count > accepted_for_items:
+                        # Nothing stored and a platform charge was made: the void
+                        # below releases its local count once the platform confirms.
+                        if accepted_for_items <= 0 and name in platform_op_names:
+                            continue
                         charging_manager._unwind_reservation(
                             name, result.charged_count - accepted_for_items
                         )
+                if accepted_for_items <= 0:
+                    # Nothing will be stored (another event of this push was capped
+                    # to zero): undo every platform charge this operation made, so
+                    # no result is billed for items that are never saved. Local
+                    # counts were unwound just above.
+                    for (name, reserved), op_id in zip(reserved_in_op, operation_ids):
+                        if reserved <= 0 or not op_id:
+                            continue
+                        # Original count: released only when the platform confirms.
+                        if op_id in uncertain_ops:
+                            await charging_manager._cancel_uncertain_charge(name, op_id, reserved)
+                        else:
+                            await charging_manager._void_charge(name, op_id, reserved)
                 if accepted_for_items > 0:
                     try:
                         await push_fn(limited_items[:accepted_for_items])
                     except Exception as push_err:
+                        # A result is billed only if it was saved: void this
+                        # operation's platform charges and unwind local
+                        # reservations, then let the caller retry the push.
                         logger.error(
-                            'Dataset push failed after charge accepted (items=%s): %s',
+                            'Dataset push failed after charge accepted (items=%s): %s; voiding the charge',
                             accepted_for_items,
                             push_err,
                         )
+                        # Only when the push certainly stored nothing (the storage API
+                        # answered with an error, or the request was never sent). A
+                        # timeout after sending may have stored the items.
+                        if not _push_certainly_failed(push_err):
+                            raise
+                        for (name, _), op_id in zip(reserved_in_op, operation_ids):
+                            kept = min(accepted_for_items, next(r.charged_count for n, r in charged_events if n == name))
+                            if kept <= 0:
+                                continue
+                            if op_id and op_id in uncertain_ops:
+                                # Never re-post a charge for items that were not stored.
+                                await charging_manager._cancel_uncertain_charge(name, op_id, kept)
+                            elif op_id:
+                                await charging_manager._void_charge(name, op_id, kept)
+                            elif is_default_dataset_item_event(name):
+                                charging_manager._unwind_reservation(name, kept)
                         raise
                 return ChargeResult(
                     chargeable_within_limit=charging_manager.compute_chargeable(),
@@ -1141,3 +1300,35 @@ async def push_data_and_charge(
             charged_count=0,
             event_charge_limit_reached=False,
         )
+
+
+def _http_status_of(err: BaseException) -> Optional[int]:
+    status = getattr(err, 'status_code', None)
+    if status is None:
+        response = getattr(err, 'response', None)
+        status = getattr(response, 'status_code', None) if response is not None else None
+    try:
+        return int(status) if status is not None else None
+    except (TypeError, ValueError):
+        return None
+
+
+def _push_certainly_failed(err: BaseException) -> bool:
+    """True when a dataset push certainly stored nothing.
+
+    The storage API answered with an error status, or the connection was never
+    made. Timeouts and resets after sending are not proof: the write may have landed.
+    """
+    import httpx
+
+    e: Optional[BaseException] = err
+    for _ in range(4):
+        if e is None:
+            break
+        status = _http_status_of(e)
+        if status is not None:
+            return status >= 400
+        if isinstance(e, (httpx.ConnectError, ConnectionRefusedError)):
+            return True
+        e = e.__cause__ or e.__context__
+    return False

scrapely/proxy_configuration.py

@@ -121,6 +121,12 @@ def _input_use_platform_proxy(actor_proxy_input: Dict[str, Any]) -> Optional[boo
     return None
 
 
+def _input_proxy_option(actor_proxy_input: Dict[str, Any], name: str) -> Any:
+    """`scrapelyProxy<name>` of the proxy input field when it is set, else the legacy `apifyProxy<name>`."""
+    value = actor_proxy_input.get(f'scrapelyProxy{name}')
+    return value if value is not None else actor_proxy_input.get(f'apifyProxy{name}')
+
+
 def _platform_env() -> Optional[Dict[str, Any]]:
     """Read platform proxy credentials from the environment (or None)."""
     hostname = os.environ.get('SCRAPELY_PROXY_HOSTNAME') or os.environ.get('APIFY_PROXY_HOSTNAME')
@@ -227,9 +233,9 @@ class ProxyConfiguration:
         # If actor_proxy_input is provided, extract settings from it
         if actor_proxy_input:
             if _input_use_platform_proxy(actor_proxy_input):
-                self._groups = actor_proxy_input.get('apifyProxyGroups', []) or []
-                self._country_code = actor_proxy_input.get('apifyProxyCountry')
-                self._subdivision_code = actor_proxy_input.get('apifyProxySubdivision')
+                self._groups = _input_proxy_option(actor_proxy_input, 'Groups') or []
+                self._country_code = _input_proxy_option(actor_proxy_input, 'Country')
+                self._subdivision_code = _input_proxy_option(actor_proxy_input, 'Subdivision')
             else:
                 self._proxy_urls = actor_proxy_input.get('proxyUrls', []) or []
                 # Tiered lists may also arrive through actor input (future-proof

{scrapely_python-0.1.2.dist-info → scrapely_python-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.2
+Version: 0.1.4
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT

{scrapely_python-0.1.2.dist-info → scrapely_python-0.1.4.dist-info}/RECORD

@@ -1,15 +1,15 @@
-scrapely/__init__.py,sha256=qBRp9YUgItQprvVgBepI-5VWc6OuQ2LQjCdSp5Xviv0,5745
+scrapely/__init__.py,sha256=nVwtGU9e9kfzoiX6MdfbK4U-p88J7eMDUkqNkpKY1Ug,5745
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
-scrapely/actor.py,sha256=wNifzbcvfH3wt0-aSaesJN_uZXNREhNfTgiORDGz7IM,75081
+scrapely/actor.py,sha256=QczHOj2GVQnMbQZ_76vjjJNHxPrinWB_WCdZ6_41l00,75126
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
-scrapely/charging.py,sha256=ckqsGbGuzpdw5PgEt6Apx3cwR46sA8V2WwUDSqSSfz8,46887
+scrapely/charging.py,sha256=MFz_Z4qJ-N3qLffleQb2BxjomygM3UMvHSBw1HabG80,56857
 scrapely/charging_client.py,sha256=tDtlPwLSVFvoRa2NBgNjyfHkAUM77SSIyYT2ACgpDm8,3710
 scrapely/configuration.py,sha256=YUnbft9JeWlVTgWalr-CgJCiyJ_e_rgoQItVPvjeWRs,15406
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
 scrapely/log.py,sha256=iVKoQdMLQXEWz1STzbSXrNUxsS_FNwiIdqoEg7CzeqU,1440
 scrapely/patch_browser_pool.py,sha256=sRAjuzKoOEgn0X8695c_9qioRALzkTWXoDlorVnK9jg,37426
 scrapely/pricing_types.py,sha256=J6wCddZd-U1MYeOtaaEdeHSsp1HBgiQLfvSInBOvhIo,5305
-scrapely/proxy_configuration.py,sha256=i6icNhei8deVK9KgCC_hdeTuSawh8WiMUOrgjvkps1s,35845
+scrapely/proxy_configuration.py,sha256=rSCXJFuUEIK31chk05FVLRQKeVA34h_pBRbr8IKNXo0,36195
 scrapely/py.typed,sha256=XtWlywqUjc6x_M2gAXlDtYXbAsYkCjKZEjFyJvaHvm4,105
 scrapely/types.py,sha256=ceVuHvhYbPai2CL3vFmXLTMoGaMR27IefJFgWhzd7ws,8900
 scrapely/webhook.py,sha256=4WOM42ypOSOs4lKtP5ofxAh6lX-7c8p5U9lOfuFzx-g,2830
@@ -37,7 +37,7 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.2.dist-info/METADATA,sha256=j0LJpAvLe7XGLBaGlKJkCq564TBZewFlGUDPgCbTB5U,3104
-scrapely_python-0.1.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.2.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.2.dist-info/RECORD,,
+scrapely_python-0.1.4.dist-info/METADATA,sha256=ACM2A5RD1zJ8JV3cEY0jZigW3dpvsjQyiwJBFvKlLv4,3104
+scrapely_python-0.1.4.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.4.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.4.dist-info/RECORD,,