scrapely-python 0.1.18__py3-none-any.whl → 0.1.19__py3-none-any.whl

scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.18'
+__version__ = '0.1.19'
 __all__ = [
     # Core
     'Actor',

scrapely/charging.py

@@ -240,6 +240,12 @@ class ChargingManager:
         self._charged_events: Dict[str, int] = {}
         self._max_total_charge_usd: Optional[Decimal] = None
         self._total_charged_usd: Decimal = Decimal('0.0')
+        # Run spend the platform counts against maxTotalChargeUsd that this SDK
+        # does not charge itself: compute/proxy/browser usage and server-billed
+        # events. Learnt from the charge endpoint's remainingBudgetNd (only ever
+        # grows), so the local budget gates on the run's true remaining budget.
+        self._platform_other_usd: Decimal = Decimal('0')
+        self._last_remaining_budget_nd: Optional[int] = None
         self._charge_lock = RLock()
         self._flush_lock = asyncio.Lock()
         # Serializes budget calculation/reservation with PPE-aware dataset pushes.
@@ -288,6 +294,27 @@ class ChargingManager:
                 self._refused_default_dataset_items,
             )
 
+    def _remaining_budget_usd(self) -> Decimal:
+        """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
+        assert self._max_total_charge_usd is not None
+        return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
+
+    def _sync_platform_budget(self) -> None:
+        """Fold the last charge response's remainingBudgetNd into the local budget.
+
+        Called after the local reservation matches what the platform accepted.
+        The platform's remaining budget = max - (our charges + usage + server
+        events), so ``other`` = max - remaining - our charges. Monotonic: usage
+        never shrinks, and a stale answer must not hand budget back.
+        """
+        nd = self._last_remaining_budget_nd
+        self._last_remaining_budget_nd = None
+        if nd is None or self._max_total_charge_usd is None or nd < 0:
+            return
+        other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
+        if other > self._platform_other_usd:
+            self._platform_other_usd = other
+
     def is_charge_limit_reached(self) -> bool:
         """Whether this pay-per-event run has used up its spending limit.
 
@@ -307,7 +334,7 @@ class ChargingManager:
         prices = [price for price in self._per_event_prices.values() if price > 0]
         if not prices:
             return False
-        return self._max_total_charge_usd - self._total_charged_usd < min(prices)
+        return self._remaining_budget_usd() < min(prices)
 
     def _mark_charge_limit_hit(self) -> None:
         if self._pricing_info is not None:
@@ -795,6 +822,7 @@ class ChargingManager:
                     idempotency_key=idempotency_key,
                     deadline=deadline,
                 )
+                self._last_remaining_budget_nd = _read_remaining_budget_nd(body)
                 return _read_accepted_count(body, count)
             except Exception as e:
                 last_error = e
@@ -957,7 +985,7 @@ class ChargingManager:
             # legacy one-event overdraw while the budget is not yet exceeded.
             chargeable_count = count
             if self._max_total_charge_usd is not None and affordability_price > 0:
-                remaining = self._max_total_charge_usd - self._total_charged_usd
+                remaining = self._remaining_budget_usd()
                 max_count_by_limit = max(0, int(remaining / affordability_price))
                 if count > max_count_by_limit:
                     chargeable_count = (
@@ -1045,6 +1073,15 @@ class ChargingManager:
                         charged_count=chargeable_count,
                         event_charge_limit_reached=self._is_event_limit_reached(event_name),
                     )
+                self._sync_platform_budget()
+                if self._max_total_charge_usd is not None and self._platform_other_usd > 0:
+                    # Usage may have used up what is left: report it on this result.
+                    result = ChargeResult(
+                        chargeable_within_limit=self._compute_chargeable(),
+                        charged_count=result.charged_count,
+                        event_charge_limit_reached=result.event_charge_limit_reached
+                        or self._is_event_limit_reached(event_name),
+                    )
             else:
                 raise ChargeUncertainError(
                     f'Charge for event {event_name!r} (count={chargeable_count}) '
@@ -1103,7 +1140,7 @@ class ChargingManager:
         if event_price is None or event_price <= 0:
             return None
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         return max(0, int(remaining / event_price))
 
     def calculate_total_charged_amount(self) -> Decimal:
@@ -1165,7 +1202,7 @@ class ChargingManager:
         if self._max_total_charge_usd is None:
             return items_count
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         max_items = int(remaining / combined_price)
 
         return min(items_count, max(0, max_items))
@@ -1226,7 +1263,7 @@ class ChargingManager:
         if self._max_total_charge_usd is None:
             return False
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         return remaining < event_price
 
     def _compute_chargeable(self) -> Dict[str, Optional[int]]:
@@ -1264,6 +1301,17 @@ def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
     )
 
 
+def _read_remaining_budget_nd(body: Any) -> Optional[int]:
+    """remainingBudgetNd from a charge response (platforms that send it), else None."""
+    if not isinstance(body, dict):
+        return None
+    data = body.get('data') if isinstance(body.get('data'), dict) else body
+    raw = data.get('remainingBudgetNd') if isinstance(data, dict) else None
+    if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0:
+        return int(raw)
+    return None
+
+
 def _read_accepted_count(body: Any, fallback: int) -> int:
     if not isinstance(body, dict):
         return fallback
@@ -1653,7 +1701,7 @@ async def push_data_with_charges(
         )
         rows = len(rows_in)
         if charging_manager._max_total_charge_usd is not None and row_cost > 0:
-            remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
+            remaining = charging_manager._remaining_budget_usd()
             rows = min(rows, max(0, int(remaining / row_cost)))
 
         ops: List[tuple] = []  # (event, charged, op_id, uncertain)

{scrapely_python-0.1.18.dist-info → scrapely_python-0.1.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.18
+Version: 0.1.19
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.18.dist-info → scrapely_python-0.1.19.dist-info}/RECORD

@@ -1,9 +1,9 @@
-scrapely/__init__.py,sha256=EHGVITjobhdZu3nhkmBmECrlftUMCsnDyLAIxdnZ_7A,5746
+scrapely/__init__.py,sha256=7AUWixuhrohpSFpDkEHo809IBKI-2u-m4ZWPMorYKSk,5746
 scrapely/_browser_locations.py,sha256=8QVVnkms2bDn3_LDYgYDR59MvFx8757l_JYnZ7xb3oI,145269
 scrapely/actor.py,sha256=Lsb4_14RCGQy-EWACnqijo7Hz5xvPKSDZcu4B8_ZrHc,81685
 scrapely/browser_configuration.py,sha256=jAZapNSHLFW05B96df4wNBtcyiiU9DA3ml0yTL1FwfA,5770
 scrapely/cdp_retry.py,sha256=hdTJEAIKhApM7IOk1DrQEl8NoflSJSmqd4RbNJ2IsgI,8540
-scrapely/charging.py,sha256=maRgijO8vM8mCb_S9lnvF99zNR_G8sB3LsubU1r6t6M,79326
+scrapely/charging.py,sha256=TxcALWN4hhwJ0i2raYmtHGA1P79lQ1bmhBMtrvNACIw,81862
 scrapely/charging_client.py,sha256=-4q-Ww7gu9EZXpM9QIn1Vd4vwAXmgL_WEKqaQIx3ViA,3961
 scrapely/configuration.py,sha256=Of-lKm7hoPGUQO8d08kV7fAy2co9QTfb8Of_8ds5PSA,17028
 scrapely/errors.py,sha256=DCAlVvE3jToZuPEG2HVDL7i8D8hLboomQ5JpSOcsXoQ,4107
@@ -39,10 +39,10 @@ scrapely/storage/_file_system/_key_value_store_client.py,sha256=4Ipgbh4qEqAkwQJ9
 scrapely/storage/_file_system/_storage_client.py,sha256=aaGJ4P0QAtIgEMl6yHpuobeWFD5M4rMC5HwhOOvuGuo,1794
 scrapely/storage_clients/__init__.py,sha256=cs1YX7JV2_GKBXeiiJgkQFali08b5_y_jH6Yw7iROAA,656
 scrapely/storages/__init__.py,sha256=4PAELhJNZ3exJ7Uouicn-bCQ58NafKjhLBixQ8ATQc4,173
-scrapely_python-0.1.18.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
-scrapely_python-0.1.18.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
-scrapely_python-0.1.18.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
-scrapely_python-0.1.18.dist-info/METADATA,sha256=3znh1pGrKvP0jNRAftMgiGSKuBG1c3g8UWJtgMYI95g,3487
-scrapely_python-0.1.18.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-scrapely_python-0.1.18.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
-scrapely_python-0.1.18.dist-info/RECORD,,
+scrapely_python-0.1.19.dist-info/licenses/LICENSE,sha256=pQGY0XwjIYh-a2HfawxHq1en53bXkFkDj2Q70jKWHr4,1088
+scrapely_python-0.1.19.dist-info/licenses/LICENSE-APACHE,sha256=N8m0FzMq8i4UbTgXZvKEZJSD-27OF8pWZYS_atz4kTU,11355
+scrapely_python-0.1.19.dist-info/licenses/NOTICE,sha256=DEsNrw6UdSIqdYYqMWVFgJnyt6jUivTD7iFU6YM_e1A,1122
+scrapely_python-0.1.19.dist-info/METADATA,sha256=PzFQ4BxF0FgSjgOvSQYimfc5v6Pq2notGIWYHCBa-iQ,3487
+scrapely_python-0.1.19.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+scrapely_python-0.1.19.dist-info/top_level.txt,sha256=T4B9UMrs4q99QSAFyD1zdXhl40KnLhBKqrDkPhOTsmY,9
+scrapely_python-0.1.19.dist-info/RECORD,,