scrapely-python 0.1.18__tar.gz → 0.1.20__tar.gz

{scrapely_python-0.1.18/src/scrapely_python.egg-info → scrapely_python-0.1.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.18
+Version: 0.1.20
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.18 → scrapely_python-0.1.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scrapely-python"
-version = "0.1.18"
+version = "0.1.20"
 description = "Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform"
 readme = "README.md"
 license = "MIT AND Apache-2.0"

{scrapely_python-0.1.18 → scrapely_python-0.1.20}/src/scrapely/__init__.py

@@ -132,7 +132,7 @@ from .errors import (
     raise_for_status,
 )
 
-__version__ = '0.1.18'
+__version__ = '0.1.20'
 __all__ = [
     # Core
     'Actor',

{scrapely_python-0.1.18 → scrapely_python-0.1.20}/src/scrapely/charging.py

@@ -240,6 +240,14 @@ class ChargingManager:
         self._charged_events: Dict[str, int] = {}
         self._max_total_charge_usd: Optional[Decimal] = None
         self._total_charged_usd: Decimal = Decimal('0.0')
+        # Run spend the platform counts against maxTotalChargeUsd that this SDK
+        # does not charge itself: compute/proxy/browser usage and server-billed
+        # events. Learnt from the charge endpoint's remainingBudgetNd (only ever
+        # grows), so the local budget gates on the run's true remaining budget.
+        self._platform_other_usd: Decimal = Decimal('0')
+        self._last_remaining_budget_nd: Optional[int] = None
+        # True once the platform reported a remaining budget (its usage is then in the local budget).
+        self._platform_budget_seen = False
         self._charge_lock = RLock()
         self._flush_lock = asyncio.Lock()
         # Serializes budget calculation/reservation with PPE-aware dataset pushes.
@@ -288,6 +296,28 @@ class ChargingManager:
                 self._refused_default_dataset_items,
             )
 
+    def _remaining_budget_usd(self) -> Decimal:
+        """maxTotalChargeUsd minus this SDK's charges and the platform's other run spend."""
+        assert self._max_total_charge_usd is not None
+        return self._max_total_charge_usd - self._total_charged_usd - self._platform_other_usd
+
+    def _sync_platform_budget(self) -> None:
+        """Fold the last charge response's remainingBudgetNd into the local budget.
+
+        Called after the local reservation matches what the platform accepted.
+        The platform's remaining budget = max - (our charges + usage + server
+        events), so ``other`` = max - remaining - our charges. Monotonic: usage
+        never shrinks, and a stale answer must not hand budget back.
+        """
+        nd = self._last_remaining_budget_nd
+        self._last_remaining_budget_nd = None
+        if nd is None or self._max_total_charge_usd is None or nd < 0:
+            return
+        self._platform_budget_seen = True
+        other = self._max_total_charge_usd - Decimal(nd) / Decimal(1_000_000_000) - self._total_charged_usd
+        if other > self._platform_other_usd:
+            self._platform_other_usd = other
+
     def is_charge_limit_reached(self) -> bool:
         """Whether this pay-per-event run has used up its spending limit.
 
@@ -307,7 +337,7 @@ class ChargingManager:
         prices = [price for price in self._per_event_prices.values() if price > 0]
         if not prices:
             return False
-        return self._max_total_charge_usd - self._total_charged_usd < min(prices)
+        return self._remaining_budget_usd() < min(prices)
 
     def _mark_charge_limit_hit(self) -> None:
         if self._pricing_info is not None:
@@ -795,6 +825,7 @@ class ChargingManager:
                     idempotency_key=idempotency_key,
                     deadline=deadline,
                 )
+                self._last_remaining_budget_nd = _read_remaining_budget_nd(body)
                 return _read_accepted_count(body, count)
             except Exception as e:
                 last_error = e
@@ -957,7 +988,7 @@ class ChargingManager:
             # legacy one-event overdraw while the budget is not yet exceeded.
             chargeable_count = count
             if self._max_total_charge_usd is not None and affordability_price > 0:
-                remaining = self._max_total_charge_usd - self._total_charged_usd
+                remaining = self._remaining_budget_usd()
                 max_count_by_limit = max(0, int(remaining / affordability_price))
                 if count > max_count_by_limit:
                     chargeable_count = (
@@ -1045,6 +1076,15 @@ class ChargingManager:
                         charged_count=chargeable_count,
                         event_charge_limit_reached=self._is_event_limit_reached(event_name),
                     )
+                self._sync_platform_budget()
+                if self._max_total_charge_usd is not None and self._platform_other_usd > 0:
+                    # Usage may have used up what is left: report it on this result.
+                    result = ChargeResult(
+                        chargeable_within_limit=self._compute_chargeable(),
+                        charged_count=result.charged_count,
+                        event_charge_limit_reached=result.event_charge_limit_reached
+                        or self._is_event_limit_reached(event_name),
+                    )
             else:
                 raise ChargeUncertainError(
                     f'Charge for event {event_name!r} (count={chargeable_count}) '
@@ -1103,7 +1143,7 @@ class ChargingManager:
         if event_price is None or event_price <= 0:
             return None
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         return max(0, int(remaining / event_price))
 
     def calculate_total_charged_amount(self) -> Decimal:
@@ -1165,7 +1205,7 @@ class ChargingManager:
         if self._max_total_charge_usd is None:
             return items_count
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         max_items = int(remaining / combined_price)
 
         return min(items_count, max(0, max_items))
@@ -1226,7 +1266,7 @@ class ChargingManager:
         if self._max_total_charge_usd is None:
             return False
 
-        remaining = self._max_total_charge_usd - self._total_charged_usd
+        remaining = self._remaining_budget_usd()
         return remaining < event_price
 
     def _compute_chargeable(self) -> Dict[str, Optional[int]]:
@@ -1264,6 +1304,17 @@ def _is_charge_limit_rejection(err: ChargeRejectedError) -> bool:
     )
 
 
+def _read_remaining_budget_nd(body: Any) -> Optional[int]:
+    """remainingBudgetNd from a charge response (platforms that send it), else None."""
+    if not isinstance(body, dict):
+        return None
+    data = body.get('data') if isinstance(body.get('data'), dict) else body
+    raw = data.get('remainingBudgetNd') if isinstance(data, dict) else None
+    if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0:
+        return int(raw)
+    return None
+
+
 def _read_accepted_count(body: Any, fallback: int) -> int:
     if not isinstance(body, dict):
         return fallback
@@ -1653,7 +1704,7 @@ async def push_data_with_charges(
         )
         rows = len(rows_in)
         if charging_manager._max_total_charge_usd is not None and row_cost > 0:
-            remaining = charging_manager._max_total_charge_usd - charging_manager._total_charged_usd
+            remaining = charging_manager._remaining_budget_usd()
             rows = min(rows, max(0, int(remaining / row_cost)))
 
         ops: List[tuple] = []  # (event, charged, op_id, uncertain)
@@ -1669,7 +1720,7 @@ async def push_data_with_charges(
                 else:
                     await charging_manager._void_charge(name, op_id, charged)
 
-        for _attempt in range(4):
+        for _attempt in range(6):
             if rows <= 0:
                 break
             ops = []
@@ -1693,10 +1744,25 @@ async def push_data_with_charges(
             if short_to is None:
                 break
             # Some event came back short: void this attempt in full and retry with
-            # the rows that event can pay for (a void restores the budget).
+            # the whole rows the budget covers at the FULL per-row cost. Sizing
+            # from the short event alone ignores the other events of the row, so
+            # the retry came back short again (appstore 64df0a3608d74af88: 20
+            # details -> 14, retry sized 14, app-item short, 0 saved). What this
+            # attempt was accepted is budget the void gives back; with the
+            # platform's remainingBudgetNd the local budget also knows the rest.
+            attempt_spent = sum(
+                (charging_manager._calculate_event_price(name) or Decimal('0')) * Decimal(charged)
+                for name, charged, _op, _unc in ops
+            )
             await undo(ops)
             ops = []
-            rows = short_to  # got < n * rows, so short_to < rows
+            fit = int(attempt_spent / row_cost) if row_cost > 0 else 0
+            if charging_manager._platform_budget_seen and charging_manager._max_total_charge_usd is not None and row_cost > 0:
+                fit = max(fit, int(charging_manager._remaining_budget_usd() / row_cost))
+            # At least one more single-row try: without the platform's remaining
+            # budget the leftover after the short answer is unknown, and the last
+            # affordable row may still fit.
+            rows = min(rows - 1, max(fit, 1))
 
         if rows <= 0 or not ops:
             await undo(ops)

{scrapely_python-0.1.18 → scrapely_python-0.1.20/src/scrapely_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapely-python
-Version: 0.1.18
+Version: 0.1.20
 Summary: Scrapely Python SDK - A toolkit for building Actors on the Scrapely platform
 Author-email: Scrapely <support@scrape.ly>
 License-Expression: MIT AND Apache-2.0

{scrapely_python-0.1.18 → scrapely_python-0.1.20}/tests/test_charge_limit_guard.py

@@ -509,3 +509,141 @@ def test_multi_event_validation() -> None:
     for bad in ({}, {'result': 0}, {'result': 1.5}, {'apify-default-dataset-item': 1}, {'': 1}, None):
         with pytest.raises(ValueError):
             _validate_row_charges(bad)
+
+
+# ---- platform usage counts against the cap (remainingBudgetNd in charge responses) ----
+
+
+class UsageApi(FakeChargeApi):
+    """Ledger whose cap also covers platform usage, reporting remainingBudgetNd like the server."""
+
+    def __init__(self, *, price: Decimal, max_usd: Decimal, usage: Decimal, send_remaining: bool = True) -> None:
+        super().__init__(price=price, max_usd=max_usd)
+        self.usage = usage
+        self.send_remaining = send_remaining
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, **_: Any) -> Dict[str, Any]:
+        count = int((json_data or {}).get('count', 1))
+        left = self.max_usd - self.spent - self.usage
+        affordable = int(left / self.price)
+        if affordable < 1:
+            raise HttpError(402, 'charge-limit-exceeded')
+        accepted = min(count, affordable)
+        self.spent += self.price * accepted
+        body: Dict[str, Any] = {'acceptedCount': accepted, 'requestedCount': count}
+        if self.send_remaining:
+            body['remainingBudgetNd'] = int((self.max_usd - self.spent - self.usage) * 1_000_000_000)
+        return body
+
+
+@pytest.mark.asyncio
+async def test_platform_usage_shrinks_the_local_budget() -> None:
+    api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'))
+    mgr = _manager(api, max_usd='0.05')
+    store = Store()
+
+    r = await mgr.charge('result', 1)
+    assert r.charged_count == 1
+    # $0.05 cap - $0.01 charged - $0.03 usage: one more event fits, not four.
+    assert mgr.calculate_max_event_charge_count_within_limit('result') == 1
+    assert mgr.is_charge_limit_reached() is False
+    pushed = await _charged_push(mgr, store, [{'n': n} for n in range(5)])
+    assert pushed.charged_count == 1 and pushed.event_charge_limit_reached is True
+    assert len(store.items) == 1
+    assert mgr.is_charge_limit_reached() is True
+    # The prepaid row from the first charge is still saved; nothing beyond it.
+    await _plain_push(mgr, store, [{'p': 1}, {'p': 2}])
+    assert len(store.items) == 2
+
+
+@pytest.mark.asyncio
+async def test_usage_learnt_from_a_charge_marks_the_limit_on_that_result() -> None:
+    api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.04'))
+    mgr = _manager(api, max_usd='0.05')
+    r = await mgr.charge('result', 1)
+    assert r.charged_count == 1
+    assert r.event_charge_limit_reached is True
+    assert mgr.is_charge_limit_reached() is True
+
+
+@pytest.mark.asyncio
+async def test_without_remaining_budget_in_responses_behaviour_is_unchanged() -> None:
+    api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('0.05'), usage=Decimal('0.03'), send_remaining=False)
+    mgr = _manager(api, max_usd='0.05')
+    await mgr.charge('result', 1)
+    assert mgr.calculate_max_event_charge_count_within_limit('result') == 4
+    assert mgr._platform_other_usd == 0
+
+
+@pytest.mark.asyncio
+async def test_platform_other_spend_never_shrinks() -> None:
+    api = UsageApi(price=Decimal('0.01'), max_usd=Decimal('1'), usage=Decimal('0.30'))
+    mgr = _manager(api, max_usd='1')
+    await mgr.charge('result', 1)
+    seen = mgr._platform_other_usd
+    assert seen == Decimal('0.30')
+    api.usage = Decimal('0.10')  # a stale or lower report
+    await mgr.charge('result', 1)
+    assert mgr._platform_other_usd == seen
+
+
+
+# ---- multi-event retry sizing: whole rows at the FULL per-row cost ----
+
+APP_PRICES = {'app-details': Decimal('0.00015'), 'app-item': Decimal('0.00005')}  # row = $0.0002
+
+
+class RemainingMultiApi(MultiApi):
+    """MultiApi that also reports remainingBudgetNd (server #725)."""
+
+    async def post(self, path: str, json_data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, **kw: Any) -> Dict[str, Any]:
+        body = await super().post(path, json_data=json_data, headers=headers, **kw)
+        if not path.endswith('/charge/void') and self.max_usd is not None:
+            body = {**body, 'remainingBudgetNd': int((self.max_usd - self.spent) * 1_000_000_000)}
+        return body
+
+
+def _app(server_max: str, remaining: bool = False):
+    api = (RemainingMultiApi if remaining else MultiApi)(APP_PRICES, Decimal(server_max))
+    return api, _manager(api, prices=dict(APP_PRICES), max_usd='1')
+
+
+@pytest.mark.asyncio
+async def test_retry_after_one_short_event_is_sized_by_the_full_row_cost() -> None:
+    """appstore 64df0a3608d74af88: details 20 -> 14 accepted; the retry must not size from 14."""
+    api, mgr = _app('0.0021')
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(20)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 10  # $0.0021 / $0.0002 per full row
+    assert len(store.items) == 10
+    assert api.spent == Decimal('0.0020')
+    assert sorted(api.ops.values()) == [('app-details', 10), ('app-item', 10)]
+
+
+@pytest.mark.asyncio
+async def test_retry_when_several_events_come_back_short() -> None:
+    """Both events of the row run short on the way down; it still converges on whole paid rows."""
+    api, mgr = _multi('1', server_max='0.0405')  # 2.53 rows of $0.016 (1 result + 3 pages)
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(6)], {'result': 1, 'page': 3}, True, store.push)
+    assert r.charged_count == 2
+    assert len(store.items) == 2
+    assert api.spent == Decimal('0.032')
+
+
+@pytest.mark.asyncio
+async def test_last_affordable_row_is_saved_without_remaining_budget() -> None:
+    api, mgr = _app('0.0002')  # exactly one row; the short answer hides the leftover
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(5)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 1 and len(store.items) == 1
+    assert api.spent == Decimal('0.0002')
+
+
+@pytest.mark.asyncio
+async def test_platform_remaining_budget_sizes_the_retry_exactly() -> None:
+    api, mgr = _app('0.00215', remaining=True)  # 10.75 rows
+    store = Store()
+    r = await push_data_with_charges(mgr, [{'n': n} for n in range(20)], {'app-details': 1, 'app-item': 1}, True, store.push)
+    assert r.charged_count == 10 and len(store.items) == 10
+    assert api.spent == Decimal('0.0020')