scrapely-python 0.1.10__tar.gz → 0.1.12__tar.gz

{scrapely_python-0.1.10/src/scrapely_python.egg-info → scrapely_python-0.1.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.10
+Version: 0.1.12
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.10 → scrapely_python-0.1.12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.10"
+version = "0.1.12"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.10 → scrapely_python-0.1.12}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.10'
+__version__ = '0.1.12'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.10 → scrapely_python-0.1.12}/src/scrapely/actor.py

@@ -24,7 +24,12 @@ from crawlee.configuration import Configuration as CrawleeConfiguration
 from crawlee.storage_clients._base import StorageClient
 from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-from .charging import ChargingManager, is_default_dataset_item_event, push_data_and_charge
+from .charging import (
+    ChargingManager,
+    is_default_dataset_item_event,
+    push_data_and_charge,
+    set_active_charging_manager,
+)
 from .configuration import Configuration
 from .crypto import decrypt_secret_inputs
 from .proxy_configuration import with_proxy_key_aliases
@@ -486,7 +491,7 @@ class Actor(metaclass=_ActorMeta):
                     if flush is not None:
                         deadline_ms = min(max(1, timeout_secs) * 1000, 30_000)
                         # The flush's own deadline ends its retries (charge POSTs back off
-                        # for up to ~45 s) a little before wait_for would cancel it.
+                        # for up to ~2 min) a little before wait_for would cancel it.
                         await asyncio.wait_for(
                             flush(max_intents=64, deadline_ms=max(500, deadline_ms - 2_000)),
                             timeout=max(1.0, deadline_ms / 1000.0),
@@ -515,6 +520,7 @@ class Actor(metaclass=_ActorMeta):
             # Clear charging + patched client so a second Actor lifecycle cannot
             # post charges to the previous run_id / credentials.
             cls._charging_manager = None
+            set_active_charging_manager(None)
             cls._apify_client_instance = None
 
             cls._default_dataset = None
@@ -1061,12 +1067,15 @@ class Actor(metaclass=_ActorMeta):
             return None
 
         client = cls.new_client(token=token, base_url=api_url)
+        # Setting the message is idempotent: a 429 (run API rate limit), 5xx or
+        # dropped connection is retried with backoff instead of failing the run.
         response = await client.put(
             f'/v2/actor-runs/{run_id}',
             json_data={
                 'statusMessage': message,
                 'isStatusMessageTerminal': is_terminal,
             },
+            idempotent=True,
         )
         body: Any = {}
         if response.content:
@@ -1222,6 +1231,8 @@ class Actor(metaclass=_ActorMeta):
             # Prefer the storage factory client (same URL/token/run as API storage).
             client = cls._client if cls._client is not None else cls.new_client()
             cls._charging_manager = ChargingManager(config, client)
+            # Storage clients refuse default-dataset writes past this run's limit.
+            set_active_charging_manager(cls._charging_manager)
         return cls._charging_manager
 
 

{scrapely_python-0.1.10 → scrapely_python-0.1.12}/src/scrapely/charging.py

@@ -9,7 +9,9 @@ never treated as retries of a previous operation.
 from __future__ import annotations
 
 import asyncio
+import contextvars
 import json
+import os
 import logging
 import time
 import uuid
@@ -47,22 +49,76 @@ CHARGE_IDEMPOTENCY_SAFE_AGE_SECS = 30 * 24 * 60 * 60
 # Soft cap on open unresolved intents. Refuse NEW billable charges when full.
 _PENDING_MAX_ENTRIES = 1024
 
-# Same-key POST attempts before leaving an intent pending / raising. The API
-# client itself retries 5xx answers and refused or reset connections of these
-# POSTs with its long backoff (0.5 s doubling to 8 s, about 45 s): re-sending
+# Same-key POST attempts before leaving an intent pending / re-queueing a void.
+# The API client itself retries 429 and 5xx answers and refused or reset
+# connections of these POSTs with exponential backoff (honouring Retry-After)
+# for its retry window (2 minutes, SCRAPELY_API_RETRY_WINDOW_SECS): re-sending
 # is safe because every attempt carries the same Idempotency-Key. These outer
 # attempts cover what the client gives up on quickly (timeouts, other
-# transport errors), with the same backoff, within _RETRY_BUDGET_SECS.
+# transport errors), within _RETRY_BUDGET_SECS. Then the charge stays pending
+# (the void is re-queued) and is retried before the next push and at exit,
+# the same as the JS SDK.
 _POST_ATTEMPTS = 8
 
-# Outer retries stop once this much time has passed since the first attempt.
-_RETRY_BUDGET_SECS = 60.0
+# Outer retries stop once this much time has passed since the first attempt
+# (the API retry window; JS SDK parity).
+_RETRY_BUDGET_SECS = 120.0
+
+
+def _is_transient_status(status: Optional[int]) -> bool:
+    """408/429 and 5xx: the platform could not take the request now; try again later."""
+    return status is not None and (status in (408, 429) or status >= 500)
+
 
 # Local-only dataset collecting one entry per logical charge operation
 # (JS ChargingManager LOCAL_CHARGING_LOG_DATASET_NAME parity).
 LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log'
 
 
+# Charge-limit safety net for the default dataset.
+#
+# Once a pay-per-event run has used up its spending limit (maxTotalChargeUsd),
+# rows written to the run's default dataset would go out unbilled. The SDK then
+# refuses such writes: nothing is stored, one warning is logged and
+# ``ChargingManager.is_charge_limit_reached()`` turns True. Pushes made by
+# ``push_data_and_charge`` (``Actor.push_data``) are decided there and marked
+# with this context variable, so the storage client does not check them twice.
+_charge_gated_push: contextvars.ContextVar[bool] = contextvars.ContextVar(
+    'scrapely_charge_gated_push', default=False
+)
+
+# The charging manager of the running Actor (set by ``Actor``); storage clients
+# consult it before writing to the default dataset.
+_active_charging_manager: Optional['ChargingManager'] = None
+
+
+def set_active_charging_manager(manager: Optional['ChargingManager']) -> None:
+    """Register the charging manager whose limit guards default-dataset writes."""
+    global _active_charging_manager
+    _active_charging_manager = manager
+
+
+def default_dataset_id_from_env() -> Optional[str]:
+    """The run's default dataset ID, as the platform passes it."""
+    return os.environ.get('ACTOR_DEFAULT_DATASET_ID') or os.environ.get('APIFY_DEFAULT_DATASET_ID')
+
+
+def refuse_default_dataset_push(item_count: int) -> bool:
+    """True when a write of ``item_count`` rows to the default dataset must be dropped.
+
+    Called by storage clients for writes that did not go through
+    ``push_data_and_charge`` (``Actor.open_dataset().push_data()``, crawler
+    ``context.push_data``, ...). Non-PPE runs and runs below their limit are
+    never refused.
+    """
+    if _charge_gated_push.get():
+        return False
+    manager = _active_charging_manager
+    if manager is None:
+        return False
+    return manager._refuse_default_dataset_push(item_count)
+
+
 def is_default_dataset_item_event(event_name: str) -> bool:
     """Check if event name is a default dataset item event."""
     return event_name in (APIFY_DEFAULT_DATASET_ITEM_EVENT, SCRAPELY_DEFAULT_DATASET_ITEM_EVENT)
@@ -209,10 +265,69 @@ class ChargingManager:
         self._last_accepted_operation_id: Optional[str] = None
         # Voids not yet confirmed by the platform; retried before the next push and at exit.
         self._pending_voids: List[tuple[str, str, int, bool]] = []
+        # Sticky: the platform refused or capped a charge (HTTP 402 / fewer
+        # events accepted) or the local budget gate trimmed one.
+        self._charge_limit_hit = False
+        # Default-dataset rows refused after the limit (one warning, then a count).
+        self._refused_default_dataset_items = 0
 
     def close(self) -> None:
         """Mark the manager closed; subsequent charge/push operations are rejected."""
         self._closed = True
+        if self._refused_default_dataset_items:
+            logger.warning(
+                '%s result(s) pushed to the default dataset after the spending limit was '
+                'reached were not saved',
+                self._refused_default_dataset_items,
+            )
+
+    def is_charge_limit_reached(self) -> bool:
+        """Whether this pay-per-event run has used up its spending limit.
+
+        True once a charge was refused or capped by the limit (the platform's
+        HTTP 402 or a partial acceptance, or the local budget gate), or once the
+        remaining budget cannot pay for even one event of the cheapest priced
+        type. From then on, pushes to the default dataset that are not paid
+        for are refused. Always False for actors without pay-per-event
+        pricing, without a limit, or with only free events.
+        """
+        if self._pricing_info is None:
+            return False
+        if self._charge_limit_hit:
+            return True
+        if self._max_total_charge_usd is None:
+            return False
+        prices = [price for price in self._per_event_prices.values() if price > 0]
+        if not prices:
+            return False
+        return self._max_total_charge_usd - self._total_charged_usd < min(prices)
+
+    def _mark_charge_limit_hit(self) -> None:
+        if self._pricing_info is not None:
+            self._charge_limit_hit = True
+
+    def _note_refused_default_dataset_items(self, count: int) -> None:
+        """Count rows dropped because of the limit; warn on the first one only."""
+        if count <= 0:
+            return
+        if not self._refused_default_dataset_items:
+            limit = (
+                f'${self._max_total_charge_usd}' if self._max_total_charge_usd is not None else 'the limit'
+            )
+            logger.warning(
+                'The spending limit of this run (maxTotalChargeUsd = %s) has been reached: '
+                'results pushed to the default dataset are no longer saved. The Actor should '
+                'stop now (check ChargeResult.event_charge_limit_reached or '
+                'Actor.get_charging_manager().is_charge_limit_reached()).',
+                limit,
+            )
+        self._refused_default_dataset_items += count
+
+    def _refuse_default_dataset_push(self, item_count: int) -> bool:
+        if item_count <= 0 or not self.is_charge_limit_reached():
+            return False
+        self._note_refused_default_dataset_items(item_count)
+        return True
 
     @property
     def charge_lock(self) -> RLock:
@@ -458,7 +573,12 @@ class ChargingManager:
         return await self._void_charge(event_name, operation_id, reserved, not_found_means_voided=True)
 
     async def _void_charge(
-        self, event_name: str, operation_id: str, reserved: int, not_found_means_voided: bool = False
+        self,
+        event_name: str,
+        operation_id: str,
+        reserved: int,
+        not_found_means_voided: bool = False,
+        deadline: Optional[float] = None,
     ) -> bool:
         """Reverse an accepted charge whose items were never stored, then unwind locally.
 
@@ -468,6 +588,8 @@ class ChargingManager:
         if self._client is None or not self._is_at_home or not self._actor_run_id:
             return False
         started = time.monotonic()
+        budget_end = started + self._RETRY_BUDGET_SECS
+        deadline = budget_end if deadline is None else min(deadline, budget_end)
         for attempt in range(self._POST_ATTEMPTS):
             try:
                 await self._client.post(
@@ -475,6 +597,7 @@ class ChargingManager:
                     json_data={},
                     headers={'Idempotency-Key': operation_id},
                     idempotent=True,
+                    retry_deadline=deadline,
                 )
                 self._unwind_reservation(event_name, reserved)
                 return True
@@ -487,24 +610,37 @@ class ChargingManager:
                     self._unwind_reservation(event_name, reserved)
                     return True
                 # 404: nothing to void; 409: already settled; 4xx auth: permanent.
-                if status is not None and 400 <= status < 500:
+                # 408/429 (rate limited) and 5xx are not answers: re-queue below.
+                if status is not None and 400 <= status < 500 and not _is_transient_status(status):
                     return False
-                if not await self._backoff_before_retry(attempt, started, err):
+                if not await self._backoff_before_retry(attempt, started, err, deadline):
                     break
         # Outcome unknown: keep the reservation and retry before the next push / at exit.
         self._pending_voids.append((event_name, operation_id, reserved, not_found_means_voided))
         logger.error('Void of charge %s (%r) not confirmed yet; will retry', operation_id, event_name)
         return False
 
-    async def _retry_pending_voids(self) -> bool:
+    async def _retry_pending_voids(self, deadline: Optional[float] = None) -> bool:
         """Retry unconfirmed voids (caller holds the operation lock).
 
-        Returns False while any earlier void is still unconfirmed.
+        With ``deadline`` (``time.monotonic()``, the exit flush) keeps retrying
+        with backoff while time is left. Returns False while any earlier void
+        is still unconfirmed.
         """
-        pending, self._pending_voids = self._pending_voids, []
-        for event_name, operation_id, reserved, not_found_means_voided in pending:
-            await self._void_charge(event_name, operation_id, reserved, not_found_means_voided)
-        return not self._pending_voids
+        from .storage.api_client import server_retry_delay
+
+        round_ = 0
+        while True:
+            pending, self._pending_voids = self._pending_voids, []
+            for event_name, operation_id, reserved, not_found_means_voided in pending:
+                await self._void_charge(event_name, operation_id, reserved, not_found_means_voided, deadline)
+            if not self._pending_voids:
+                return True
+            left = 0.0 if deadline is None else deadline - time.monotonic()
+            if left <= 0:
+                return False
+            await asyncio.sleep(min(server_retry_delay(round_, None), left))
+            round_ += 1
 
     def _quarantine_stale_pending(self) -> None:
         """Remove intents older than the safe replay window (fail closed, no replay).
@@ -551,7 +687,7 @@ class ChargingManager:
             },
             headers={'Idempotency-Key': idempotency_key},
             idempotent=True,
-            **({'retry_deadline': deadline} if deadline is not None else {}),
+            retry_deadline=deadline,
         )
         if isinstance(response, dict):
             return response
@@ -569,17 +705,17 @@ class ChargingManager:
     ) -> bool:
         """Sleep before the next same-key attempt; False when no attempt is left.
 
-        A 5xx answer reaching this point was already retried by the API client
-        with its long backoff, so it ends the attempts. Timeouts and other
-        errors without an answer back off the same way (0.5 s doubling to 8 s)
-        while attempts and the retry budget last.
+        A 429/5xx answer reaching this point was already retried by the API
+        client for the whole budget, so it ends the attempts. Timeouts, 408 and
+        other errors back off the same way while attempts and the retry budget
+        last.
         """
         from .storage.api_client import server_retry_delay
 
         if attempt + 1 >= self._POST_ATTEMPTS:
             return False
         status = _http_status_of(err)
-        if status is not None and status >= 500:
+        if status is not None and (status == 429 or status >= 500):
             return False
         delay = server_retry_delay(attempt, None)
         if time.monotonic() - started + delay > self._RETRY_BUDGET_SECS:
@@ -609,6 +745,9 @@ class ChargingManager:
         """
         last_error: Optional[BaseException] = None
         started = time.monotonic()
+        # The client's own retries count against the same budget.
+        budget_end = started + self._RETRY_BUDGET_SECS
+        deadline = budget_end if deadline is None else min(deadline, budget_end)
         for attempt in range(self._POST_ATTEMPTS):
             try:
                 body = await self._post_charge_once(
@@ -657,11 +796,10 @@ class ChargingManager:
         """Caller must hold _operation_lock."""
         if not self._is_at_home or self._client is None or not self._actor_run_id:
             return 0
-        await self._retry_pending_voids()
-
-        import time
-
         deadline = None if deadline_ms is None else (time.monotonic() + max(0.0, deadline_ms) / 1000.0)
+        # Voids first: an unconfirmed one means the user may be billed for items
+        # that were never stored. At exit (with a deadline) keep retrying them.
+        await self._retry_pending_voids(deadline)
         confirmed_total = 0
         async with self._flush_lock:
             while True:
@@ -688,6 +826,8 @@ class ChargingManager:
                             deadline=deadline,
                         )
                     except ChargeRejectedError as err:
+                        if _is_charge_limit_rejection(err):
+                            self._mark_charge_limit_hit()
                         self._unwind_reservation(event_name, count)
                         self._drop_pending(idempotency_key)
                         logger.warning(
@@ -699,6 +839,7 @@ class ChargingManager:
                         continue
                     if accepted is not None:
                         if accepted < count:
+                            self._mark_charge_limit_hit()
                             self._unwind_reservation(event_name, count - accepted)
                         self._drop_pending(idempotency_key)
                         confirmed += 1
@@ -784,6 +925,9 @@ class ChargingManager:
                         else 0
                     )
 
+            if chargeable_count < count:
+                self._mark_charge_limit_hit()
+
             if chargeable_count <= 0:
                 return ChargeResult(
                     chargeable_within_limit=self._compute_chargeable(),
@@ -835,7 +979,9 @@ class ChargingManager:
                     count=chargeable_count,
                     idempotency_key=idempotency_key,
                 )
-            except ChargeRejectedError:
+            except ChargeRejectedError as rejected:
+                if _is_charge_limit_rejection(rejected):
+                    self._mark_charge_limit_hit()
                 # Server definitively rejected: unwind local reservation.
                 self._drop_pending(idempotency_key)
                 self._unwind_reservation(event_name, chargeable_count)
@@ -849,6 +995,8 @@ class ChargingManager:
                 if accepted > 0:
                     self._last_accepted_operation_id = idempotency_key
                 if accepted < chargeable_count:
+                    # The platform capped the charge at the run's limit.
+                    self._mark_charge_limit_hit()
                     self._unwind_reservation(event_name, chargeable_count - accepted)
                     chargeable_count = accepted
                     result = ChargeResult(
@@ -1066,6 +1214,15 @@ class ChargingManager:
 
 
 
+def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
+    """Whether a rejected charge was refused because the run's limit is used up."""
+    return (
+        err.status_code == 402
+        or err.reason == 'charge-limit-exceeded'
+        or 'charge-limit-exceeded' in str(err).lower()
+    )
+
+
 def _read_accepted_count(body: Any, fallback: int) -> int:
     if not isinstance(body, dict):
         return fallback
@@ -1166,6 +1323,24 @@ async def push_data_and_charge(
             if fallback_event:
                 resolved_event = fallback_event
 
+        # Charge-limit safety net: once the run's spending limit is used up, a
+        # default-dataset push that does not pay per row (no event, or only
+        # free events) stores nothing. Paid pushes go on below, where the
+        # budget cap stores only the rows that are actually charged.
+        if (
+            is_ppe
+            and is_default_dataset
+            and items
+            and charging_manager.is_charge_limit_reached()
+            and not _push_is_paid_per_item(charging_manager, event_name, resolved_event)
+        ):
+            charging_manager._note_refused_default_dataset_items(len(items))
+            return ChargeResult(
+                chargeable_within_limit=charging_manager.compute_chargeable(),
+                charged_count=0,
+                event_charge_limit_reached=True,
+            )
+
         limited_count = len(items)
         if is_ppe and resolved_event:
             # Keep the explicit event separate from the implicit default-dataset
@@ -1177,6 +1352,10 @@ async def push_data_and_charge(
             )
 
         limited_items = items[:limited_count] if limited_count < len(items) else items
+        if is_ppe and is_default_dataset and limited_count < len(items):
+            # Rows beyond what the budget can pay for are dropped.
+            charging_manager._mark_charge_limit_hit()
+            charging_manager._note_refused_default_dataset_items(len(items) - limited_count)
 
         if not limited_items:
             return ChargeResult(
@@ -1302,9 +1481,13 @@ async def push_data_and_charge(
                             await charging_manager._cancel_uncertain_charge(name, op_id, reserved)
                         else:
                             await charging_manager._void_charge(name, op_id, reserved)
+                if is_default_dataset and accepted_for_items < len(limited_items):
+                    charging_manager._note_refused_default_dataset_items(
+                        len(limited_items) - accepted_for_items
+                    )
                 if accepted_for_items > 0:
                     try:
-                        await push_fn(limited_items[:accepted_for_items])
+                        await _gated_push(push_fn, limited_items[:accepted_for_items])
                     except Exception as push_err:
                         # A result is billed only if it was saved: void this
                         # operation's platform charges and unwind local
@@ -1344,7 +1527,7 @@ async def push_data_and_charge(
 
 
         # No billable events — still persist the budget-limited items.
-        await push_fn(limited_items)
+        await _gated_push(push_fn, limited_items)
         return ChargeResult(
             chargeable_within_limit=charging_manager.compute_chargeable(),
             charged_count=0,
@@ -1352,6 +1535,23 @@ async def push_data_and_charge(
         )
 
 
+def _push_is_paid_per_item(
+    charging_manager: ChargingManager, event_name: Optional[str], resolved_event: Optional[str]
+) -> bool:
+    """Whether every row of this push is charged a positive price."""
+    names = {name for name in (event_name, resolved_event) if name}
+    return any((charging_manager._calculate_event_price(name) or Decimal('0')) > 0 for name in names)
+
+
+async def _gated_push(push_fn: Any, items: List[Dict[str, Any]]) -> Any:
+    """Run a push already decided by push_data_and_charge (storage skips its own check)."""
+    token = _charge_gated_push.set(True)
+    try:
+        return await push_fn(items)
+    finally:
+        _charge_gated_push.reset(token)
+
+
 def _http_status_of(err: BaseException) -> Optional[int]:
     status = getattr(err, 'status_code', None)
     if status is None:

{scrapely_python-0.1.10 → scrapely_python-0.1.12}/src/scrapely/charging_client.py

@@ -43,7 +43,11 @@ class _ChargingManagerHooks:
         from .charging import has_default_dataset_item_event_in_pricing
 
         pricing_info = self._charging_manager.get_pricing_info()
-        return has_default_dataset_item_event_in_pricing(pricing_info.per_event_prices or {})
+        # Past the spending limit, route default-dataset pushes through
+        # push_data_and_charge too, which refuses rows nobody pays for.
+        return has_default_dataset_item_event_in_pricing(
+            pricing_info.per_event_prices or {}
+        ) or bool(getattr(self._charging_manager, 'is_charge_limit_reached', lambda: False)())
 
     async def push_and_charge(
         self,